SPB Git

Repository Public

TransformDraftTests.swift 9.0 KB 143 lines swift

1import Foundation2import Testing3import SheetCore4import SheetData5@testable import FeatureData67@Suite struct StepDraftTests {8    /// At least one step of every kind, with the variants of each parameter.9    static let samples: [TransformStep] = [10        .filter(FilterGroup(.any, [FilterCondition("region", .isIn, [.string("North"), .string("a, b"), .null]),11                                   FilterCondition("amount", .between, [.double(10.5), .int(20)]),12                                   FilterCondition("product", .contains, .string("42"), caseInsensitive: true),13                                   FilterCondition("order_date", .greaterOrEqual, .date(year: 2024, month: 3, day: 1)),14                                   FilterCondition("id", .isNull)])),15        .filter(FilterGroup(.all, [])),16        .sort([SortSpec("region"), SortSpec("amount", ascending: false, nullsFirst: true), SortSpec("id", ascending: true, nullsFirst: false)]),17        .renameColumn(from: "amount", to: "total amount"),18        .removeColumns(["id", "product"]),19        .selectColumns(["region", "amount"]),20        .reorderColumns(["amount", "id"]),21        .changeType(column: "id", to: .string, onError: .null, format: nil),22        .changeType(column: "amount", to: .decimal(precision: 12, scale: 3), onError: .fail, format: nil),23        .changeType(column: "order_date", to: .timestamp, onError: .null, format: "%Y-%m-%d"),24        .changeType(column: "quantity", to: .boolean, onError: .null, format: nil),25        .changeType(column: "quantity", to: .integer, onError: .null, format: nil),26        .changeType(column: "quantity", to: .double, onError: .null, format: nil),27        .changeType(column: "region", to: .date, onError: .fail, format: "%d/%m/%Y"),28        .changeType(column: "region", to: .time, onError: .null, format: nil),29        .splitColumn(column: "product", by: .delimiter("-"), into: ["a", "b"], keepOriginal: true),30        .splitColumn(column: "product", by: .positions([3, 5]), into: ["p1", "p2", "p3"], keepOriginal: false),31        .splitColumn(column: "product", by: .regex("(\\w)(\\w+)"), into: ["first", "rest"], keepOriginal: false),32        .mergeColumns(columns: ["region", "product"], separator: " / ", into: "label", keepOriginals: false),33        .join(dataset: "customers", kind: .left, keys: [JoinKey("id", "customer_id"), JoinKey("region")], suffix: "_c"),34        .join(dataset: "customers", kind: .anti, keys: [JoinKey("id")], suffix: ""),35        .groupBy(keys: ["region", "product"], aggregations: [Aggregation(.sum, "amount", as: "total"), Aggregation(.countRows), Aggregation(.median, "quantity"),36                                                              Aggregation(.distinctCount, "id", as: "ids")]),37        .groupBy(keys: [], aggregations: [Aggregation(.average, "amount")]),38        .pivot(rowKeys: ["region"], column: "product", value: "amount", aggregation: .average),39        .unpivot(columns: ["quantity", "amount"], nameColumn: "measure", valueColumn: "value"),40        .removeDuplicates(columns: []),41        .removeDuplicates(columns: ["region", "product"]),42        .fillMissing(columns: ["amount"], strategy: .constant(.double(0.5))),43        .fillMissing(columns: ["region"], strategy: .constant(.string("unknown"))),44        .fillMissing(columns: ["amount"], strategy: .previous),45        .fillMissing(columns: ["amount"], strategy: .next),46        .fillMissing(columns: ["amount"], strategy: .mean),47        .fillMissing(columns: ["amount"], strategy: .median),48        .dropMissing(columns: [], mode: .any),49        .dropMissing(columns: ["amount", "region"], mode: .all),50        .replaceValues(column: "region", replacements: [Replacement(.string("North"), .string("N")), Replacement(.null, .string("?")), Replacement(.int(3), .double(3.5))], matchSubstring: false),51        .replaceValues(column: "region", replacements: [Replacement(.string("or"), .string("OR"))], matchSubstring: true),52        .parseDate(column: "region", format: "%d/%m/%Y", asTimestamp: false, onError: .null),53        .parseDate(column: "region", format: "%Y-%m-%d %H:%M", asTimestamp: true, onError: .fail),54        .extractText(column: "product", operation: .regex(pattern: "(\\d+)", group: 1), into: "digits"),55        .extractText(column: "product", operation: .before("-"), into: nil),56        .extractText(column: "product", operation: .after(":"), into: "tail"),57        .extractText(column: "product", operation: .substring(start: 2, length: 4), into: "mid"),58        .extractText(column: "product", operation: .substring(start: 0, length: nil), into: nil),59        .extractText(column: "product", operation: .length, into: "len"),60        .extractText(column: "product", operation: .trim, into: nil),61        .extractText(column: "product", operation: .uppercase, into: nil),62        .extractText(column: "product", operation: .lowercase, into: "lower"),63        .calculatedColumn(name: "revenue", expression: "quantity * amount * (1 - 0.1)"),64        .limit(250),65        .sample(.rows(100), seed: 7),66        .sample(.percent(12.5), seed: -3),67        .appendDataset("orders_2023"),68    ]6970    @Test func samplesCoverEveryStepKind() {71        #expect(Set(Self.samples.map { StepKind($0) }) == Set(StepKind.allCases))72        #expect(StepKind.allCases.count == 23)73        #expect(Set(StepKind.Category.allCases.flatMap(\.kinds)) == Set(StepKind.allCases))74    }7576    @Test(arguments: samples) func draftRoundTripsWithoutLoss(_ step: TransformStep) throws {77        let draft = StepDraft(step)78        #expect(draft.kind == StepKind(step))79        #expect(try draft.makeStep() == step)80        // every value of the draft belongs to a declared field, so the generated form can show all of it81        let keys = Set(draft.kind.fields.map(\.key))82        #expect(Set(draft.values.keys).isSubset(of: keys), "\(draft.kind): \(Set(draft.values.keys).subtracting(keys))")83    }8485    @Test func formsDeclareEveryParameter() {86        for kind in StepKind.allCases {87            #expect(!kind.fields.isEmpty, "\(kind)")88            #expect(!kind.title.isEmpty && !kind.systemImage.isEmpty)89            #expect(Set(kind.fields.map(\.key)).count == kind.fields.count)90            let draft = StepDraft(kind: kind, schema: [])91            for field in kind.fields { #expect(draft.values[field.key] != nil, "\(kind).\(field.key) has no default") }92            for field in kind.fields { if let when = field.when { #expect(kind.fields.contains { $0.key == when.key }) } }93        }94    }9596    @Test func reportsTheFieldThatIsWrong() {97        var draft = StepDraft(.limit(10))98        draft.values["count"] = .text("many")99        #expect(throws: DraftError.self) { try draft.makeStep() }100        draft = StepDraft(.calculatedColumn(name: "x", expression: "a + b"))101        draft.values["expression"] = .text("a; DROP TABLE t")102        #expect(throws: DraftError(SQL.expressionProblem("a; DROP TABLE t")!)) { try draft.makeStep() }103        draft.values["expression"] = .text("a + (b")104        #expect(throws: DraftError.self) { try draft.makeStep() }105        draft = StepDraft(.renameColumn(from: "a", to: "b"))106        draft.values["to"] = .text("  ")107        #expect(throws: DraftError.self) { try draft.makeStep() }108        draft = StepDraft(.sample(.percent(5), seed: 1))109        draft.values["percent"] = .text("250")110        #expect(throws: DraftError.self) { try draft.makeStep() }111        draft = StepDraft(.selectColumns(["a"]))112        draft.values["columns"] = .list([])113        #expect(throws: DraftError.self) { try draft.makeStep() }114        draft = StepDraft(.join(dataset: "d", kind: .inner, keys: [JoinKey("a")], suffix: "_r"))115        draft.values["keys"] = .rows([])116        #expect(throws: DraftError.self) { try draft.makeStep() }117    }118119    @Test func visibilityFollowsTheChosenVariant() {120        var draft = StepDraft(.sample(.rows(10), seed: 1))121        let fields = Dictionary(uniqueKeysWithValues: StepKind.sample.fields.map { ($0.key, $0) })122        #expect(draft.isVisible(fields["rows"]!) && !draft.isVisible(fields["percent"]!) && draft.isVisible(fields["seed"]!))123        draft.values["mode"] = .text("percent")124        #expect(!draft.isVisible(fields["rows"]!) && draft.isVisible(fields["percent"]!))125    }126127    @Test func defaultStepsAreValidForTheSchemaTheyAreAddedTo() throws {128        let dir = TestFiles.directory()129        defer { try? FileManager.default.removeItem(at: dir) }130        let engine = try DataEngine()131        try engine.importFile(TestFiles.ordersCSV(200, in: dir), as: "orders")132        try engine.importFile(TestFiles.ordersCSV(50, in: dir, name: "more.csv"), as: "more")133        let schema = try engine.schema(of: "orders")134        for kind in StepKind.allCases {135            let step = try StepDraft(kind: kind, schema: schema, datasets: ["more"]).makeStep()136            #expect(StepKind(step) == kind)137            let diagnostics = TransformCompiler.validate(TransformPipeline(name: "p", source: .dataset("orders"), steps: [step]), engine: engine)138            #expect(diagnostics.count == 2)139            #expect(diagnostics.last?.status == .ok, "\(kind): \(diagnostics.last?.message ?? "")")140        }141    }142}143
Files main
Menu

Theme