Repository Public
1import Foundation2import Testing3import SheetCore4import SheetData5@testable import FeatureData67@Suite struct StepDraftTests {8 /// At least one step of every kind, with the variants of each parameter.9 static let samples: [TransformStep] = [10 .filter(FilterGroup(.any, [FilterCondition("region", .isIn, [.string("North"), .string("a, b"), .null]),11 FilterCondition("amount", .between, [.double(10.5), .int(20)]),12 FilterCondition("product", .contains, .string("42"), caseInsensitive: true),13 FilterCondition("order_date", .greaterOrEqual, .date(year: 2024, month: 3, day: 1)),14 FilterCondition("id", .isNull)])),15 .filter(FilterGroup(.all, [])),16 .sort([SortSpec("region"), SortSpec("amount", ascending: false, nullsFirst: true), SortSpec("id", ascending: true, nullsFirst: false)]),17 .renameColumn(from: "amount", to: "total amount"),18 .removeColumns(["id", "product"]),19 .selectColumns(["region", "amount"]),20 .reorderColumns(["amount", "id"]),21 .changeType(column: "id", to: .string, onError: .null, format: nil),22 .changeType(column: "amount", to: .decimal(precision: 12, scale: 3), onError: .fail, format: nil),23 .changeType(column: "order_date", to: .timestamp, onError: .null, format: "%Y-%m-%d"),24 .changeType(column: "quantity", to: .boolean, onError: .null, format: nil),25 .changeType(column: "quantity", to: .integer, onError: .null, format: nil),26 .changeType(column: "quantity", to: .double, onError: .null, format: nil),27 .changeType(column: "region", to: .date, onError: .fail, format: "%d/%m/%Y"),28 .changeType(column: "region", to: .time, onError: .null, format: nil),29 .splitColumn(column: "product", by: .delimiter("-"), into: ["a", "b"], keepOriginal: true),30 .splitColumn(column: "product", by: .positions([3, 5]), into: ["p1", "p2", "p3"], keepOriginal: false),31 .splitColumn(column: "product", by: .regex("(\\w)(\\w+)"), into: ["first", "rest"], keepOriginal: false),32 .mergeColumns(columns: ["region", "product"], separator: " / ", into: "label", keepOriginals: false),33 .join(dataset: "customers", kind: .left, keys: [JoinKey("id", "customer_id"), JoinKey("region")], suffix: "_c"),34 .join(dataset: "customers", kind: .anti, keys: [JoinKey("id")], suffix: ""),35 .groupBy(keys: ["region", "product"], aggregations: [Aggregation(.sum, "amount", as: "total"), Aggregation(.countRows), Aggregation(.median, "quantity"),36 Aggregation(.distinctCount, "id", as: "ids")]),37 .groupBy(keys: [], aggregations: [Aggregation(.average, "amount")]),38 .pivot(rowKeys: ["region"], column: "product", value: "amount", aggregation: .average),39 .unpivot(columns: ["quantity", "amount"], nameColumn: "measure", valueColumn: "value"),40 .removeDuplicates(columns: []),41 .removeDuplicates(columns: ["region", "product"]),42 .fillMissing(columns: ["amount"], strategy: .constant(.double(0.5))),43 .fillMissing(columns: ["region"], strategy: .constant(.string("unknown"))),44 .fillMissing(columns: ["amount"], strategy: .previous),45 .fillMissing(columns: ["amount"], strategy: .next),46 .fillMissing(columns: ["amount"], strategy: .mean),47 .fillMissing(columns: ["amount"], strategy: .median),48 .dropMissing(columns: [], mode: .any),49 .dropMissing(columns: ["amount", "region"], mode: .all),50 .replaceValues(column: "region", replacements: [Replacement(.string("North"), .string("N")), Replacement(.null, .string("?")), Replacement(.int(3), .double(3.5))], matchSubstring: false),51 .replaceValues(column: "region", replacements: [Replacement(.string("or"), .string("OR"))], matchSubstring: true),52 .parseDate(column: "region", format: "%d/%m/%Y", asTimestamp: false, onError: .null),53 .parseDate(column: "region", format: "%Y-%m-%d %H:%M", asTimestamp: true, onError: .fail),54 .extractText(column: "product", operation: .regex(pattern: "(\\d+)", group: 1), into: "digits"),55 .extractText(column: "product", operation: .before("-"), into: nil),56 .extractText(column: "product", operation: .after(":"), into: "tail"),57 .extractText(column: "product", operation: .substring(start: 2, length: 4), into: "mid"),58 .extractText(column: "product", operation: .substring(start: 0, length: nil), into: nil),59 .extractText(column: "product", operation: .length, into: "len"),60 .extractText(column: "product", operation: .trim, into: nil),61 .extractText(column: "product", operation: .uppercase, into: nil),62 .extractText(column: "product", operation: .lowercase, into: "lower"),63 .calculatedColumn(name: "revenue", expression: "quantity * amount * (1 - 0.1)"),64 .limit(250),65 .sample(.rows(100), seed: 7),66 .sample(.percent(12.5), seed: -3),67 .appendDataset("orders_2023"),68 ]6970 @Test func samplesCoverEveryStepKind() {71 #expect(Set(Self.samples.map { StepKind($0) }) == Set(StepKind.allCases))72 #expect(StepKind.allCases.count == 23)73 #expect(Set(StepKind.Category.allCases.flatMap(\.kinds)) == Set(StepKind.allCases))74 }7576 @Test(arguments: samples) func draftRoundTripsWithoutLoss(_ step: TransformStep) throws {77 let draft = StepDraft(step)78 #expect(draft.kind == StepKind(step))79 #expect(try draft.makeStep() == step)80 // every value of the draft belongs to a declared field, so the generated form can show all of it81 let keys = Set(draft.kind.fields.map(\.key))82 #expect(Set(draft.values.keys).isSubset(of: keys), "\(draft.kind): \(Set(draft.values.keys).subtracting(keys))")83 }8485 @Test func formsDeclareEveryParameter() {86 for kind in StepKind.allCases {87 #expect(!kind.fields.isEmpty, "\(kind)")88 #expect(!kind.title.isEmpty && !kind.systemImage.isEmpty)89 #expect(Set(kind.fields.map(\.key)).count == kind.fields.count)90 let draft = StepDraft(kind: kind, schema: [])91 for field in kind.fields { #expect(draft.values[field.key] != nil, "\(kind).\(field.key) has no default") }92 for field in kind.fields { if let when = field.when { #expect(kind.fields.contains { $0.key == when.key }) } }93 }94 }9596 @Test func reportsTheFieldThatIsWrong() {97 var draft = StepDraft(.limit(10))98 draft.values["count"] = .text("many")99 #expect(throws: DraftError.self) { try draft.makeStep() }100 draft = StepDraft(.calculatedColumn(name: "x", expression: "a + b"))101 draft.values["expression"] = .text("a; DROP TABLE t")102 #expect(throws: DraftError(SQL.expressionProblem("a; DROP TABLE t")!)) { try draft.makeStep() }103 draft.values["expression"] = .text("a + (b")104 #expect(throws: DraftError.self) { try draft.makeStep() }105 draft = StepDraft(.renameColumn(from: "a", to: "b"))106 draft.values["to"] = .text(" ")107 #expect(throws: DraftError.self) { try draft.makeStep() }108 draft = StepDraft(.sample(.percent(5), seed: 1))109 draft.values["percent"] = .text("250")110 #expect(throws: DraftError.self) { try draft.makeStep() }111 draft = StepDraft(.selectColumns(["a"]))112 draft.values["columns"] = .list([])113 #expect(throws: DraftError.self) { try draft.makeStep() }114 draft = StepDraft(.join(dataset: "d", kind: .inner, keys: [JoinKey("a")], suffix: "_r"))115 draft.values["keys"] = .rows([])116 #expect(throws: DraftError.self) { try draft.makeStep() }117 }118119 @Test func visibilityFollowsTheChosenVariant() {120 var draft = StepDraft(.sample(.rows(10), seed: 1))121 let fields = Dictionary(uniqueKeysWithValues: StepKind.sample.fields.map { ($0.key, $0) })122 #expect(draft.isVisible(fields["rows"]!) && !draft.isVisible(fields["percent"]!) && draft.isVisible(fields["seed"]!))123 draft.values["mode"] = .text("percent")124 #expect(!draft.isVisible(fields["rows"]!) && draft.isVisible(fields["percent"]!))125 }126127 @Test func defaultStepsAreValidForTheSchemaTheyAreAddedTo() throws {128 let dir = TestFiles.directory()129 defer { try? FileManager.default.removeItem(at: dir) }130 let engine = try DataEngine()131 try engine.importFile(TestFiles.ordersCSV(200, in: dir), as: "orders")132 try engine.importFile(TestFiles.ordersCSV(50, in: dir, name: "more.csv"), as: "more")133 let schema = try engine.schema(of: "orders")134 for kind in StepKind.allCases {135 let step = try StepDraft(kind: kind, schema: schema, datasets: ["more"]).makeStep()136 #expect(StepKind(step) == kind)137 let diagnostics = TransformCompiler.validate(TransformPipeline(name: "p", source: .dataset("orders"), steps: [step]), engine: engine)138 #expect(diagnostics.count == 2)139 #expect(diagnostics.last?.status == .ok, "\(kind): \(diagnostics.last?.message ?? "")")140 }141 }142}143