SPB Git

Repository Public

RealModelTests.swift 19.4 KB 233 lines swift

1import Foundation2import Testing3import SheetCore4import Audit5@testable import SheetIntelligence67/// Tests against the REAL on-device model. Nothing here is scripted.8///9/// They are opt-in: every test that asks the model to generate (`oversizedPromptMapsToContextWindowExceeded`,10/// `fullEvaluation`) runs only when the environment contains exactly `ORIX_AI_EVAL=1`. With the variable unset, or11/// set to anything else (`0` included), they are reported as skipped, the model is never called and `EVALUATION.md`12/// is never rewritten. `availabilityIsReported` always runs: it only asks the system whether the model is available13/// and generates nothing. With `ORIX_AI_EVAL=1` on a machine without the model, the tests print the reason and return.14@Suite(.serialized) struct RealModelTests {15    /// True only on explicit opt-in.16    static var evaluationRequested: Bool { ProcessInfo.processInfo.environment["ORIX_AI_EVAL"] == "1" }17    static let skipComment: Comment = "real-model tests are opt-in: set ORIX_AI_EVAL=1 (this rewrites EVALUATION.md)"1819    struct Case { let request: String; let expected: Value }2021    static var cases: [Case] {22        let u = (0..<20).map { Double(Fixtures.units($0)) }, p = (0..<20).map { Fixtures.price($0) }, r = (0..<20).map { Fixtures.revenue($0) }23        func region(_ i: Int) -> String { Fixtures.regions[i % 4] }24        func product(_ i: Int) -> String { Fixtures.products[i % 3] }25        func sum(_ v: [Double], _ keep: (Int) -> Bool) -> Double { (0..<20).filter(keep).reduce(0) { $0 + v[$1] } }26        let total = r.reduce(0, +)27        let sortedPrices = p.sorted()28        let widget = (0..<20).filter { product($0) == "Widget" }29        return [30            Case(request: "Total revenue", expected: .number(total)),31            Case(request: "Average number of units sold", expected: .number(u.reduce(0, +) / 20)),32            Case(request: "Highest unit price", expected: .number(p.max()!)),33            Case(request: "Lowest revenue", expected: .number(r.min()!)),34            Case(request: "How many sales are in the North region?", expected: .number(5)),35            Case(request: "Total revenue for the West region", expected: .number(sum(r) { region($0) == "West" })),36            Case(request: "Number of sales with more than 20 units", expected: .number(Double(u.filter { $0 > 20 }.count))),37            Case(request: "Total units sold of the product Gadget", expected: .number(sum(u) { product($0) == "Gadget" })),38            Case(request: "Average revenue of Widget sales", expected: .number(widget.reduce(0) { $0 + r[$1] } / Double(widget.count))),39            Case(request: "Revenue of the first sale with tax added, using the TaxRate name", expected: .number(r[0] * 1.15)),40            Case(request: "Number of different products", expected: .number(3)),41            Case(request: "Region of the sale with the highest revenue", expected: .text(region(r.firstIndex(of: r.max()!)!))),42            Case(request: "Total revenue for Widget in the North region", expected: .number(sum(r) { region($0) == "North" && product($0) == "Widget" })),43            Case(request: "Median unit price", expected: .number((sortedPrices[9] + sortedPrices[10]) / 2)),44            Case(request: "Share of total revenue that comes from the East region", expected: .number(sum(r) { region($0) == "East" } / total)),45            Case(request: "Revenue of the first sale rounded to the nearest whole number", expected: .number(r[0].rounded())),46            Case(request: "Largest number of units in a single sale in the South region", expected: .number((0..<20).filter { region($0) == "South" }.map { u[$0] }.max()!)),47            Case(request: "Second largest revenue", expected: .number(r.sorted()[18])),48            Case(request: "Number of sales dated in February 2026", expected: .number(Double((0..<20).filter { $0 / 7 + 1 == 2 }.count))),49            Case(request: "The text High if total units are above 300, otherwise Low", expected: .text(u.reduce(0, +) > 300 ? "High" : "Low")),50        ]51    }5253    static func matches(_ value: Value?, _ expected: Value) -> Bool {54        guard let value = value?.scalar else { return false }55        switch (value, expected) {56        case (.number(let a), .number(let b)): return abs(a - b) <= 1e-6 * max(1, abs(b))57        case (.text(let a), .text(let b)): return a.lowercased() == b.lowercased()58        default: return false59        }60    }6162    static func progress(_ text: String) { FileHandle.standardError.write(Data(("[eval] " + text + "\n").utf8)) }6364    /// The reason the real model cannot be used here, or nil when it can.65    func unavailableReason() async -> String? {66        if case .unavailable(let reason) = await FoundationModelsBackend().availability {67            print("REAL MODEL TESTS SKIPPED — on-device model unavailable: \(reason.code): \(reason.explanation)")68            return reason.code69        }70        return nil71    }7273    @Test func availabilityIsReported() async {74        let a = await Intelligence().availability()75        print("On-device model availability on this machine: \(a)")76        if case .unavailable(let r) = a { #expect(!r.explanation.isEmpty) }77    }7879    @Test(.enabled(if: RealModelTests.evaluationRequested, RealModelTests.skipComment)) func oversizedPromptMapsToContextWindowExceeded() async {80        guard Self.evaluationRequested else { return }81        guard await unavailableReason() == nil else { return }82        var big = "Region,Product,Units,Revenue\n"83        for i in 0..<3000 { big += "R\(i % 7),Widget \(i),\(i * 3 % 97),\(Double(i) * 12.5)\n" }84        do {85            _ = try await FoundationModelsBackend().respond(instructions: "Summarise the table.", prompt: big, options: GenerationOptions())86            Issue.record("an oversized prompt was accepted")87        } catch { #expect(error as? IntelligenceError == .contextWindowExceeded) }88    }8990    @Test(.enabled(if: RealModelTests.evaluationRequested, RealModelTests.skipComment)) func fullEvaluation() async throws {91        guard Self.evaluationRequested else { print("Evaluation skipped (ORIX_AI_EVAL is not 1)."); return }92        if let reason = await unavailableReason() { print("EVALUATION.md not written (\(reason))."); return }93        let backend = FoundationModelsBackend()94        let availability = await backend.availability95        let english = Intelligence(backend: backend, locale: Locale(identifier: "en_US"))96        let context = Fixtures.context(active: "H2")97        var md = "# SheetIntelligence — measured evaluation\n\n"98        md += "Generated by `ORIX_AI_EVAL=1 swift test` (`RealModelTests.fullEvaluation`) against the real on-device model. Nothing below is scripted or edited by hand. The evaluation is opt-in: a plain `swift test` (variable unset, or any value other than `1`) skips every test that calls the model and leaves this file untouched.\n\n"99        md += "- Date: \(ISO8601DateFormatter().string(from: Date()))\n- macOS: \(ProcessInfo.processInfo.operatingSystemVersionString)\n"100        md += "- Model availability: \(availability)\n- Function registry: \(FunctionRegistry.standard.count) functions (core + FunctionsNumeric + FunctionsText + FunctionsLookup)\n"101        md += "- Workbook: synthetic sales table `Sales!A1:F21` (Region, Product, Units, Unit Price, Revenue, Date), sheet `Params` with the name `TaxRate`; active cell `H2`\n- Decoding: greedy (temperature 0) for every task, streamed prose included; formulas and Python as plain text lines, review / charts / plan with guided generation\n- An earlier run of the same 20 requests with formulas generated inside guided JSON is kept verbatim in `EVALUATION-run1-guided-json.md` (13 / 20 correct); see \"Changes between runs\" at the end\n\n"102103        // 1. formula suggestions104        var rows = ""105        var suggestions = 0, valid = 0, firstPassValidRequests = 0, anyValidRequests = 0, topCorrect = 0, anyCorrect = 0, repairs = 0, repairsCorrect = 0, failures = 0106        var seconds = 0.0107        for (n, c) in Self.cases.enumerated() {108            let start = Date()109            var out: [FormulaSuggestion] = []110            var note = ""111            do { out = try await english.suggestFormulas(request: c.request, context: context) } catch { failures += 1; note = "request failed: \((error as? IntelligenceError)?.code ?? "\(error)")" }112            seconds += Date().timeIntervalSince(start)113            Self.progress("request \(n + 1)/\(Self.cases.count) in \(String(format: "%.1f", Date().timeIntervalSince(start))) s: \(out.count) suggestions \(note)")114            suggestions += out.count115            valid += out.filter(\.validation.isValid).count116            if out.contains(where: { $0.validation.isValid && !$0.isRepair }) { firstPassValidRequests += 1 }117            if out.contains(where: \.validation.isValid) { anyValidRequests += 1 }118            if let first = out.first, first.validation.isValid, Self.matches(first.validation.value, c.expected) { topCorrect += 1 }119            let correct = out.contains { $0.validation.isValid && Self.matches($0.validation.value, c.expected) }120            if correct { anyCorrect += 1 }121            if out.contains(where: \.isRepair) { repairs += 1; if out.contains(where: { $0.isRepair && $0.validation.isValid && Self.matches($0.validation.value, c.expected) }) { repairsCorrect += 1 } }122            let listing = out.map { s -> String in123                let verdict = s.validation.isValid ? (Self.matches(s.validation.value, c.expected) ? "valid, correct (\(s.validation.preview ?? ""))" : "valid, WRONG value (\(s.validation.preview ?? ""))")124                    : "invalid: \(s.validation.reason ?? "")"125                return "`\(s.rawText.replacingOccurrences(of: "|", with: "\\|").replacingOccurrences(of: "\n", with: " "))`\(s.isRepair ? " (repair)" : "") — \(verdict)"126            }.joined(separator: "<br>")127            rows += "| \(n + 1) | \(c.request) | \(FormulaValidator.preview(c.expected)) | \(correct ? "yes" : "no") | \(listing.isEmpty ? note : listing) |\n"128        }129        let total = Self.cases.count130        md += "## Formula suggestions (\(total) fixed requests)\n\n"131        md += "| Measure | Result |\n|---|---|\n"132        md += "| Requests answered without error | \(total - failures) / \(total) |\n"133        md += "| Suggestions produced | \(suggestions) |\n| Suggestions that passed validation (parse, functions, references, engine evaluation) | \(valid) / \(suggestions) |\n"134        md += "| Requests with at least one valid suggestion on the first pass | \(firstPassValidRequests) / \(total) |\n"135        md += "| Requests with at least one valid suggestion (after the single repair round) | \(anyValidRequests) / \(total) |\n"136        md += "| Requests whose first listed suggestion is valid and computes the expected value | \(topCorrect) / \(total) |\n"137        md += "| Requests where some valid suggestion computes the expected value | \(anyCorrect) / \(total) |\n"138        md += "| Repair rounds triggered / repairs that produced the expected value | \(repairs) / \(repairsCorrect) |\n"139        md += "| Mean time per request | \(String(format: "%.1f", seconds / Double(total))) s |\n\n"140        md += "Expected values are computed independently in Swift from the generator of the synthetic data. A suggestion that validates but computes another value counts as wrong.\n\n"141        md += "| # | Request | Expected | Correct | Suggestions (raw model text — verdict) |\n|---|---|---|---|---|\n" + rows + "\n"142        #expect(failures < total)143144        // 2. other tasks, observed once each145        md += "## Other tasks (one observation each)\n\n"146        let (wb, engine) = Fixtures.sales()147        engine.set(sheet: 0, "H2", .formula(try FormulaParser.parse("=SUMIF(A2:A21,\"North\",E2:E21)/SUM(E2:E21)")))148        engine.set(sheet: 0, "H3", .formula(try FormulaParser.parse("=E2/(C2-C2)")))149        engine.set(sheet: 0, "E9", .formula(try FormulaParser.parse("=C9*14")))150        _ = engine.recalculate()151        func block(_ text: String) -> String { text.split(separator: "\n", omittingEmptySubsequences: false).map { "> " + $0 }.joined(separator: "\n") + "\n\n" }152        func observe(_ title: String, _ body: () async throws -> String) async {153            md += "### \(title)\n\n"154            Self.progress(title)155            do { md += try await body() } catch { md += "Failed: \((error as? IntelligenceError)?.errorDescription ?? "\(error)")\n\n" }156        }157        let formula = try #require(ContextBuilder.formulaContext(engine: engine, sheet: 0, cell: CellRef(a1: "H2")!))158        await observe("Explain a formula (English)") {159            var text = ""160            for try await s in try await english.explainFormula(formula) { text = s }161            let check = NumberVerifier.check(text, allowed: formula.steps.compactMap { Value.parseNumber($0.value) } + formula.references.compactMap { Value.parseNumber($0.value) } + [Value.parseNumber(formula.result)].compactMap { $0 }, ignoring: [formula.formula] + formula.references.map(\.address))162            #expect(!text.isEmpty)163            return "Formula `\(formula.formula)`, engine result \(formula.result).\n\n" + block(text) + "Numbers not found among the engine's values: \(check.unverified.isEmpty ? "none" : check.unverified.joined(separator: ", ")).\n\n"164        }165        let french = Intelligence(backend: backend, locale: Locale(identifier: "fr_CA"))166        await observe("Explain a formula (app locale fr-CA)") {167            let language = await french.answerLanguage()168            var text = ""169            for try await s in try await french.explainFormula(formula) { text = s }170            let words = Set(text.lowercased().split { !$0.isLetter }.map(String.init))171            let looksFrench = words.intersection(["la", "le", "les", "des", "est", "cette", "dans", "pour", "somme", "cellule", "cellules"]).count >= 3172            #expect(!text.isEmpty)173            return "`supportsLocale(fr_CA)` → answer language \(language). Answer looks French (stop-word heuristic): \(looksFrench).\n\n" + block(text)174        }175        let profile = ContextBuilder.tableProfile(workbook: wb, sheet: 0, range: RangeRef(a1: "A1:F21")!)176        await observe("Describe data") {177            let d = try await english.describeData(profile)178            return block(d.text) + "Numbers verified against the profile: \(d.verifiedNumberCount). Flagged as unverified: \(d.unverifiedNumbers.isEmpty ? "none" : d.unverifiedNumbers.joined(separator: ", ")).\n\n"179        }180        await observe("Suggest charts") {181            let charts = try await english.suggestCharts(profile)182            #expect(charts.allSatisfy { c in (c.y + [c.x].compactMap { $0 }).allSatisfy { profile.column(named: $0) != nil } })183            return charts.map { "- \($0.kind.rawValue): x = \($0.x ?? "—"), y = \($0.y.joined(separator: ", ")) — \($0.rationale)\n" }.joined() + "\n\(charts.count) accepted after validation.\n\n"184        }185        await observe("Transformation plan (goal: \"total revenue per region, highest first\")") {186            let steps = try await english.transformationPlan(goal: "total revenue per region, highest first", profile: profile)187            return steps.map { "\($0.id). \($0.operation.rawValue) [\($0.columns.joined(separator: ", "))]\($0.newColumns.isEmpty ? "" : " → [\($0.newColumns.joined(separator: ", "))]"): \($0.detail)\($0.issues.isEmpty ? "" : " — ISSUES: \($0.issues.joined(separator: " "))")\n" }.joined() + "\n\(steps.filter(\.isValid).count) of \(steps.count) steps consistent with the profile.\n\n"188        }189        let findings = ModelAuditor.audit(workbook: wb, engine: engine).findings.map(AuditFindingSummary.init)190        let audited = ContextBuilder.sheetContext(workbook: wb, sheet: 0, selection: RangeRef(a1: "H2")!, active: CellRef(a1: "H2")!)191        await observe("Review auditor findings (\(findings.count) findings from the deterministic auditor)") {192            let review = try await english.reviewForErrors(findings: findings, context: audited)193            #expect(review.items.allSatisfy { item in findings.contains { $0.id == item.findingID } })194            return block(review.overview) + review.items.map { "\($0.priority). `\($0.findingID)` — \($0.explanation) Fix: \($0.suggestedFix)\n" }.joined()195                + "\nInvented ids rejected: \(review.rejectedIDs.count).\n\n"196        }197        await observe("Explain statistics (descriptive summary of Units)") {198            let table = [["Statistic", "Units"], ["Count", "20"], ["Mean", "15.25"], ["Median", "15"], ["Std. deviation (sample)", "6.71"], ["Min", "5"], ["Max", "27"]]199            var text = ""200            for try await s in try await english.explainStatistics(table, kind: "Descriptive statistics") { text = s }201            let check = NumberVerifier.check(text, allowed: ReportRendering.numbers(table))202            return block(text) + "Numbers not in the table: \(check.unverified.isEmpty ? "none" : check.unverified.joined(separator: ", ")).\n\n"203        }204        await observe("Python help (request: \"total revenue per region written next to the table\")") {205            let p = try await english.pythonHelp(request: "Compute total revenue per region and write the result starting at cell H1", context: audited, currentCode: nil)206            return "```python\n\(p.code)\n```\n\nSyntax check (compile only, never executed): \(p.syntax). Unknown `orix` calls: \(p.unknownAPICalls.isEmpty ? "none" : p.unknownAPICalls.joined(separator: ", ")).\n\n"207        }208        await observe("Document a sheet (model paragraph only; the rest of the document is deterministic)") {209            var doc = ""210            for try await s in try await english.documentSheet(audited, audit: findings) { doc = s }211            let purpose = doc.components(separatedBy: SheetDocumenter.purposeHeading).last?.components(separatedBy: "\n## ").first ?? ""212            #expect(doc.contains("`=C2*D2`"))213            return block(purpose.trimmingCharacters(in: .whitespacesAndNewlines))214        }215        md += """216        ## Changes between runs217218        Run 1 (`EVALUATION-run1-guided-json.md`, same machine, same day, same 20 requests): 38 / 57 suggestions valid, 13 / 20 requests correct.219        Its failures showed three general defects, which were changed before this run (the 20 requests and their expected values were not changed, and no request-specific wording was added to the prompts):220221        1. Formulas were generated inside JSON strings (guided generation). Formulas contain double quotes, and the answers were cut at the first quote (`=COUNTIFS(F2:F21, `) or used a single quote instead. Formulas (and Python) are now requested as plain `FORMULA:` / `WHY:` lines and parsed strictly.222        2. The prompt listed the signatures of the functions "closest to the request" (keyword match), which pushed the model towards exotic functions (`AGGREGATE`, `CHOOSECOLS`, `EVEN`). It now lists function names only and asks for the simplest formula; it also states two general syntax facts (criteria are quoted text such as ">20"; dates are written with `DATE`).223        3. The model was asked for "up to 3 different formulas", which produced valid formulas that answer a different question. It is now asked for the best formula, with alternatives only when they answer the same request.224225        No further prompt changes were made after this run.226227        """228        let url = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("EVALUATION.md")229        try md.write(to: url, atomically: true, encoding: .utf8)230        print("EVALUATION.md written: \(anyCorrect)/\(total) requests correct, \(valid)/\(suggestions) suggestions valid.")231    }232}233
Files main
Menu

Theme