SPB Git

Repository Public

AssistantModelTests.swift 17.4 KB 332 lines swift

1import Foundation2import Testing3import SheetCore4import SheetUI5import SheetIntelligence6import FunctionsNumeric7import FunctionsText8import FunctionsLookup9@testable import FeatureIntelligence1011@MainActor enum Demo {12    static let registered: Bool = {13        NumericFunctions.register(); TextFunctions.register(); LookupFunctions.register(); FinancialFunctions.register()14        return true15    }()1617    /// A1:C6 table (Region, Units, Revenue = Units × 2) and a formula in E2.18    static func session() -> WorkbookSession {19        _ = registered20        let session = WorkbookSession(workbook: Workbook(sheetNames: ["Data"]), locale: Locale(identifier: "en_US"))21        let parser = session.parser22        session.perform("Setup") { e in23            let rows = [["Region", "Units", "Revenue"], ["North", "5", "=B2*2"], ["South", "7", "=B3*2"], ["North", "9", "=B4*2"], ["East", "11", "=B5*2"], ["West", "13", "=B6*2"]]24            for (r, row) in rows.enumerated() { for (c, text) in row.enumerated() { try? e.setInput(sheet: 0, row: r, col: c, text, parser: parser) } }25            try? e.setInput(sheet: 0, row: 1, col: 4, "=SUM(C2:C6)", parser: parser)26        }27        return session28    }2930    static func model(_ responses: [String], availability: IntelligenceAvailability = .available(modelDescription: "test"), languages: Set<String> = ["en", "fr"],31                      locale: String = "en_US") -> (AssistantModel, ScriptedBackend) {32        let backend = ScriptedBackend(responses: responses, availability: availability, languages: languages)33        return (AssistantModel(intelligence: Intelligence(backend: backend, locale: Locale(identifier: locale))), backend)34    }3536    static func select(_ session: WorkbookSession, _ a1: String) { session.select(RangeRef(a1: a1)!, active: RangeRef(a1: a1)!.topLeft) }37}3839@MainActor @Suite struct AssistantModelTests {40    @Test func unavailableDisablesAIButKeepsComputedFacts() async {41        let session = Demo.session()42        let (model, backend) = Demo.model([], availability: .unavailable(reason: .appleIntelligenceNotEnabled))43        await model.refreshAvailability()44        #expect(model.availability == .unavailable(reason: .appleIntelligenceNotEnabled))45        #expect(!model.isAvailable)46        for task in AssistantTask.allCases {47            model.task = task48            model.request = "something"49            Demo.select(session, task == .explain ? "E2" : "B3")50            model.refresh(session, force: true)51            #expect(model.blocker(for: session) == IntelligenceLocalization.string("The on-device model is unavailable."), "\(task)")52            #expect(model.run(session) == nil)53            #expect(model.phase == .idle)54        }55        #expect(backend.requests.isEmpty)56        // deterministic facts are still there57        model.task = .explain; Demo.select(session, "E2"); model.refresh(session, force: true)58        #expect(model.formula?.formula == "=SUM(C2:C6)" && model.formula?.result == "90")59        model.task = .describe; Demo.select(session, "B3"); model.refresh(session, force: true)60        #expect(model.profile?.rowCount == 5 && model.profile?.columns.map(\.name) == ["Region", "Units", "Revenue"])61        model.task = .document; model.refresh(session, force: true)62        #expect(model.documentation?.contains("`=B2*2`") == true)63        model.task = .review; model.refresh(session, force: true); model.runAuditor(session)64        #expect(model.findings != nil)65    }6667    @Test func explainStreamsAndChecksNumbers() async {68        let session = Demo.session()69        Demo.select(session, "E2")70        let (model, backend) = Demo.model(["It adds the five revenue cells and gets 90, or roughly 95 with luck."])71        await model.refreshAvailability()72        model.task = .explain73        model.refresh(session)74        #expect(model.blocker(for: session) == nil)75        #expect(model.sharedContext.contains("=SUM(C2:C6)"))76        await model.run(session)?.value77        #expect(model.phase == .finished)78        #expect(model.outputTask == .explain)79        #expect(model.answer.hasPrefix("It adds the five revenue cells"))80        #expect(model.unverifiedNumbers == ["95"])81        #expect(backend.requests.count == 1 && backend.requests[0].prompt == model.sharedContext)82        #expect(model.debugMarker == "SCRIPTED PREVIEW")83    }8485    @Test func explainNeedsAFormula() async {86        let session = Demo.session()87        Demo.select(session, "A2")88        let (model, _) = Demo.model(["x"])89        await model.refreshAvailability()90        model.refresh(session)91        #expect(model.formula == nil)92        #expect(model.blocker(for: session) == IntelligenceLocalization.string("Select a cell that contains a formula."))93        #expect(model.run(session) == nil)94    }9596    @Test func suggestionsAreInsertedOnlyOnRequestAsOneUndoableStep() async {97        let session = Demo.session()98        Demo.select(session, "E4")99        let (model, _) = Demo.model(["FORMULA: =SUMIF(A2:A6,\"North\",C2:C6)\nWHY: North revenue\nFORMULA: =NOPE(1)\nWHY: bad"])100        await model.refreshAvailability()101        model.task = .suggest102        #expect(model.blocker(for: session) == IntelligenceLocalization.string("Type a request first."))103        model.request = "revenue for North"104        await model.run(session)?.value105        #expect(model.suggestions.count == 2)106        #expect(model.suggestions[0].validation.value == .number(28))107        #expect(!model.suggestions[1].validation.isValid)108        // nothing is written until the user asks109        #expect(session.sheet.cell(3, 4) == nil)110        let undoDepth = session.engine.undoLabel111        #expect(!model.insert(model.suggestions[1], into: session))112        #expect(session.sheet.cell(3, 4) == nil)113        // the selection moves away; the formula still goes where it was validated114        Demo.select(session, "A1")115        #expect(model.insert(model.suggestions[0], into: session))116        #expect(session.sheet.value(3, 4) == .number(28))117        #expect(session.engine.inputText(sheet: 0, row: 3, col: 4) == "=SUMIF(A2:A6,\"North\",C2:C6)")118        #expect(session.selection.active == CellRef(a1: "E4")!)119        #expect(session.engine.undoLabel != undoDepth)120        session.undo()121        #expect(session.sheet.cell(3, 4)?.hasContent != true)122    }123124    @Test func insertRevalidatesAgainstTheCurrentWorkbook() async {125        let session = Demo.session()126        Demo.select(session, "E4")127        let (model, _) = Demo.model(["FORMULA: =SUM(Extra!A1:A3)"])128        await model.refreshAvailability()129        model.task = .suggest; model.request = "sum"130        session.addSheet()131        _ = session.renameSheet(at: 1, to: "Extra")132        session.activateSheet(0)133        Demo.select(session, "E4")134        await model.run(session)?.value135        #expect(model.suggestions.first?.validation.isValid == true)136        session.removeSheet(at: 1)137        #expect(!model.insert(model.suggestions[0], into: session))138        #expect(session.sheet.cell(3, 4)?.hasContent != true)139        #expect(session.statusMessage == IntelligenceLocalization.string("The formula is no longer valid for this sheet. Nothing was inserted."))140    }141142    @Test func describeUsesTheProfileAndFlagsUnverifiedNumbers() async {143        let session = Demo.session()144        Demo.select(session, "B3")145        let (model, backend) = Demo.model(["There are 5 rows. Units go from 5 to 13. Sales rose 40%."])146        await model.refreshAvailability()147        model.task = .describe148        model.refresh(session)149        await model.run(session)?.value150        #expect(model.unverifiedNumbers == ["40%"])151        #expect(backend.requests[0].prompt == model.profile?.rendered())152        #expect(model.dataRange == RangeRef(a1: "A1:C6")!)153    }154155    @Test func sampleValuesCanBeWithheld() async {156        let session = Demo.session()157        Demo.select(session, "E4")158        let (model, backend) = Demo.model(["FORMULA: =SUM(C2:C6)"])159        await model.refreshAvailability()160        model.task = .suggest; model.request = "sum"161        model.includeSampleValues = false162        model.refresh(session, force: true)163        #expect(model.sharedContext.isEmpty)      // not built until someone looks at it164        model.wantsContextPreview = true165        model.refresh(session)166        #expect(!model.sharedContext.contains("North"))167        #expect(model.sharedContext.contains("\"Region\" (text)"))168        await model.run(session)?.value169        #expect(!backend.requests[0].prompt.contains("North |"))170        #expect(backend.requests[0].prompt.hasPrefix(model.sharedContext))171    }172173    @Test func chartsPlanAndReviewAreFiltered() async {174        let session = Demo.session()175        Demo.select(session, "B3")176        let charts = "{\"charts\": [{\"kind\": \"column\", \"title\": \"t\", \"x\": \"Region\", \"y\": [\"Revenue\"], \"rationale\": \"r\"}, {\"kind\": \"line\", \"x\": \"Month\", \"y\": [\"Revenue\"]}]}"177        let plan = "{\"steps\": [{\"operation\": \"groupBy\", \"columns\": [\"Region\"], \"newColumns\": [\"Total\"], \"detail\": \"sum Revenue\"}, {\"operation\": \"sort\", \"columns\": [\"Ghost\"], \"detail\": \"desc\"}]}"178        let (model, _) = Demo.model([charts, plan])179        await model.refreshAvailability()180        model.task = .charts181        await model.run(session)?.value182        #expect(model.charts.count == 1 && model.charts[0].kind == .column)183        model.task = .plan; model.request = "totals per region"184        await model.run(session)?.value185        #expect(model.charts.isEmpty)       // previous output is cleared186        #expect(model.plan.count == 2 && model.plan[0].isValid && !model.plan[1].isValid)187        #expect(model.planText().hasPrefix("1. Group by [Region] → [Total]: sum Revenue"))188        var copied = ""189        model.copy = { copied = $0 }190        model.copyText(model.planText(), session: session)191        #expect(copied == model.planText())192    }193194    @Test func reviewBuildsOnAuditorFindingsOnly() async {195        let session = Demo.session()196        let parser = session.parser197        session.perform("Break") { e in try? e.setInput(sheet: 0, row: 7, col: 4, "=B2/0", parser: parser) }198        Demo.select(session, "A1")199        let (probe, _) = Demo.model([])200        probe.task = .review201        probe.runAuditor(session)202        let real = try! #require(probe.findings?.first { $0.severity == "error" })203        let json = "{\"overview\": \"One error.\", \"items\": [{\"findingId\": \"made:Up!A1\", \"priority\": 1, \"explanation\": \"x\", \"suggestedFix\": \"y\"}, {\"findingId\": \"\(real.id)\", \"priority\": 2, \"explanation\": \"Division by zero.\", \"suggestedFix\": \"Check the divisor.\"}]}"204        let (model, backend) = Demo.model([json])205        await model.refreshAvailability()206        model.task = .review207        await model.run(session)?.value208        #expect(model.findings?.contains(real) == true)      // the auditor ran first209        #expect(model.review?.items.map(\.findingID) == [real.id])210        #expect(model.review?.rejectedIDs == ["made:Up!A1"])211        #expect(backend.requests[0].prompt.contains("id: \(real.id)"))212    }213214    @Test func pythonIsNeverRunOnlyCopiedForTheStudio() async {215        let session = Demo.session()216        let (model, _) = Demo.model(["```python\nimport orix\nprint(orix.active_sheet().name)\n```\nPrints the name."])217        await model.refreshAvailability()218        model.task = .python; model.request = "print the sheet name"219        await model.run(session)?.value220        let suggestion = try! #require(model.python)221        #expect(suggestion.code.hasPrefix("import orix"))222        var copied = ""223        model.copy = { copied = $0 }224        model.openInPythonStudio(suggestion, session: session)225        #expect(copied == suggestion.code)226        #expect(session.studioPane == "python")227        #expect(session.statusMessage?.contains("It has not been run") == true)228    }229230    @Test func failuresAreSurfacedAndLeaveTheWorkbookAlone() async {231        let session = Demo.session()232        Demo.select(session, "E2")233        let before = session.engine.changeCount234        for error in [IntelligenceError.guardrailViolation, .refusal(""), .unsupportedLanguage, .invalidResponse("x")] {235            let backend = ScriptedBackend(results: [.failure(error)])236            let model = AssistantModel(intelligence: Intelligence(backend: backend, locale: Locale(identifier: "en_US")))237            await model.refreshAvailability()238            model.task = .explain239            await model.run(session)?.value240            #expect(model.phase == .failed(error))241            #expect(model.answer.isEmpty)242            #expect(!error.panelMessage.isEmpty)243        }244        #expect(session.engine.changeCount == before)245    }246247    @Test func languageFallbackIsReported() async {248        let (fr, _) = Demo.model(["x"], languages: ["en", "fr"], locale: "fr_CA")249        await fr.refreshAvailability()250        #expect(fr.language == .appLanguage(code: "fr"))251        let (pl, _) = Demo.model(["x"], languages: ["en"], locale: "pl_PL")252        await pl.refreshAvailability()253        #expect(pl.language == .englishFallback(requested: "pl"))254    }255256    @Test func stopKeepsWhatWasWritten() async {257        let session = Demo.session()258        Demo.select(session, "E2")259        let (model, _) = Demo.model(["one two three"])260        await model.refreshAvailability()261        model.task = .explain262        let job = model.run(session)263        #expect(model.phase == .running)264        model.stop()265        #expect(model.phase == .idle)266        await job?.value267        #expect(model.phase == .idle || model.phase == .finished)268    }269270    @Test func registrationAddsThePaneAndActions() {271        IntelligenceFeature.register()272        let pane = Workspace.shared.panes(.inspector).first { $0.id == "assistant" }273        #expect(pane?.systemImage == "sparkles")274        let ids = Set(Workspace.shared.actions.map(\.id))275        #expect(ids.isSuperset(of: ["intelligence.explain", "intelligence.suggest", "intelligence.describe", "intelligence.document"]))276        #expect(Workspace.shared.actions(.tools).contains { $0.title == IntelligenceLocalization.string("Suggest a Formula…") })277        let session = Demo.session()278        IntelligenceFeature.setIntelligenceFactory { Intelligence(backend: ScriptedBackend(responses: ["x"]), locale: Locale(identifier: "en_US")) }279        Workspace.shared.actions.first { $0.id == "intelligence.suggest" }?.perform(session)280        #expect(session.inspectorPane == "assistant")281        #expect(IntelligenceFeature.model(for: session).task == .suggest)282        #expect(IntelligenceFeature.model(for: session).focusRequest)283        #expect(IntelligenceFeature.model(for: session) === IntelligenceFeature.model(for: session))284    }285286    @Test func markdownParsing() {287        let blocks = MarkdownText.parse("# Title\n\nSome *text*\nwrapped.\n\n- one\n- two\n\n| A | B |\n|---|---|\n| `=C2\\|1` | 2 |\n")288        #expect(blocks == [.heading(1, "Title"), .paragraph("Some *text* wrapped."), .bullet("one"), .bullet("two"), .table([["A", "B"], ["=C2|1", "2"]])])289    }290}291292@Suite struct LocalizationTests {293    static var catalog: [String: Any] {294        let url = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()295            .appendingPathComponent("Sources/FeatureIntelligence/Resources/Localizable.xcstrings")296        let json = try! JSONSerialization.jsonObject(with: Data(contentsOf: url)) as! [String: Any]297        return json["strings"] as! [String: Any]298    }299300    static var usedKeys: [String] {301        let dir = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("Sources/FeatureIntelligence")302        var keys = Set<String>()303        let regex = try! NSRegularExpression(pattern: #"\bL\("((?:[^"\\]|\\.)*)""#)304        for file in try! FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil) where file.pathExtension == "swift" {305            let text = try! String(contentsOf: file, encoding: .utf8)306            for m in regex.matches(in: text, range: NSRange(text.startIndex..., in: text)) { keys.insert(String(text[Range(m.range(at: 1), in: text)!])) }307        }308        return keys.sorted()309    }310311    @Test func everyKeyHasACanadianFrenchTranslation() {312        let catalog = Self.catalog313        #expect(Self.usedKeys.count > 150)314        for key in Self.usedKeys {315            let entry = ((catalog[key] as? [String: Any])?["localizations"] as? [String: Any])?["fr-CA"] as? [String: Any]316            let value = (entry?["stringUnit"] as? [String: Any])?["value"] as? String317            #expect(value?.isEmpty == false, "missing fr-CA for \(key)")318            // same format specifiers, in the same order319            func specifiers(_ s: String) -> [String] { s.components(separatedBy: "%").dropFirst().compactMap { $0.first.map(String.init) }.filter { "@d".contains($0) } }320            if let value { #expect(specifiers(value) == specifiers(key), "specifiers differ for \(key)") }321        }322        #expect(Set(catalog.keys) == Set(Self.usedKeys))323    }324325    @Test func frenchBundleIsBuilt() throws {326        let path = try #require(IntelligenceLocalization.bundle.path(forResource: "fr-CA", ofType: "lproj"))327        let fr = try #require(Bundle(path: path))328        #expect(fr.localizedString(forKey: "Runs on this Mac · nothing leaves the device", value: nil, table: nil) == "S’exécute sur ce Mac · rien ne quitte l’appareil")329        #expect(fr.localizedString(forKey: "Generated on device — verify before relying on it", value: nil, table: nil) == "Généré sur l’appareil — à vérifier avant de s’y fier")330    }331}332
Files main
Menu

Theme