Repository Public
1import Foundation2import Testing3import SheetCore4import Audit5@testable import SheetIntelligence67/// Tests against the REAL on-device model. Nothing here is scripted.8///9/// They are opt-in: every test that asks the model to generate (`oversizedPromptMapsToContextWindowExceeded`,10/// `fullEvaluation`) runs only when the environment contains exactly `ORIX_AI_EVAL=1`. With the variable unset, or11/// set to anything else (`0` included), they are reported as skipped, the model is never called and `EVALUATION.md`12/// is never rewritten. `availabilityIsReported` always runs: it only asks the system whether the model is available13/// and generates nothing. With `ORIX_AI_EVAL=1` on a machine without the model, the tests print the reason and return.14@Suite(.serialized) struct RealModelTests {15 /// True only on explicit opt-in.16 static var evaluationRequested: Bool { ProcessInfo.processInfo.environment["ORIX_AI_EVAL"] == "1" }17 static let skipComment: Comment = "real-model tests are opt-in: set ORIX_AI_EVAL=1 (this rewrites EVALUATION.md)"1819 struct Case { let request: String; let expected: Value }2021 static var cases: [Case] {22 let u = (0..<20).map { Double(Fixtures.units($0)) }, p = (0..<20).map { Fixtures.price($0) }, r = (0..<20).map { Fixtures.revenue($0) }23 func region(_ i: Int) -> String { Fixtures.regions[i % 4] }24 func product(_ i: Int) -> String { Fixtures.products[i % 3] }25 func sum(_ v: [Double], _ keep: (Int) -> Bool) -> Double { (0..<20).filter(keep).reduce(0) { $0 + v[$1] } }26 let total = r.reduce(0, +)27 let sortedPrices = p.sorted()28 let widget = (0..<20).filter { product($0) == "Widget" }29 return [30 Case(request: "Total revenue", expected: .number(total)),31 Case(request: "Average number of units sold", expected: .number(u.reduce(0, +) / 20)),32 Case(request: "Highest unit price", expected: .number(p.max()!)),33 Case(request: "Lowest revenue", expected: .number(r.min()!)),34 Case(request: "How many sales are in the North region?", expected: .number(5)),35 Case(request: "Total revenue for the West region", expected: .number(sum(r) { region($0) == "West" })),36 Case(request: "Number of sales with more than 20 units", expected: .number(Double(u.filter { $0 > 20 }.count))),37 Case(request: "Total units sold of the product Gadget", expected: .number(sum(u) { product($0) == "Gadget" })),38 Case(request: "Average revenue of Widget sales", expected: .number(widget.reduce(0) { $0 + r[$1] } / Double(widget.count))),39 Case(request: "Revenue of the first sale with tax added, using the TaxRate name", expected: .number(r[0] * 1.15)),40 Case(request: "Number of different products", expected: .number(3)),41 Case(request: "Region of the sale with the highest revenue", expected: .text(region(r.firstIndex(of: r.max()!)!))),42 Case(request: "Total revenue for Widget in the North region", expected: .number(sum(r) { region($0) == "North" && product($0) == "Widget" })),43 Case(request: "Median unit price", expected: .number((sortedPrices[9] + sortedPrices[10]) / 2)),44 Case(request: "Share of total revenue that comes from the East region", expected: .number(sum(r) { region($0) == "East" } / total)),45 Case(request: "Revenue of the first sale rounded to the nearest whole number", expected: .number(r[0].rounded())),46 Case(request: "Largest number of units in a single sale in the South region", expected: .number((0..<20).filter { region($0) == "South" }.map { u[$0] }.max()!)),47 Case(request: "Second largest revenue", expected: .number(r.sorted()[18])),48 Case(request: "Number of sales dated in February 2026", expected: .number(Double((0..<20).filter { $0 / 7 + 1 == 2 }.count))),49 Case(request: "The text High if total units are above 300, otherwise Low", expected: .text(u.reduce(0, +) > 300 ? "High" : "Low")),50 ]51 }5253 static func matches(_ value: Value?, _ expected: Value) -> Bool {54 guard let value = value?.scalar else { return false }55 switch (value, expected) {56 case (.number(let a), .number(let b)): return abs(a - b) <= 1e-6 * max(1, abs(b))57 case (.text(let a), .text(let b)): return a.lowercased() == b.lowercased()58 default: return false59 }60 }6162 static func progress(_ text: String) { FileHandle.standardError.write(Data(("[eval] " + text + "\n").utf8)) }6364 /// The reason the real model cannot be used here, or nil when it can.65 func unavailableReason() async -> String? {66 if case .unavailable(let reason) = await FoundationModelsBackend().availability {67 print("REAL MODEL TESTS SKIPPED — on-device model unavailable: \(reason.code): \(reason.explanation)")68 return reason.code69 }70 return nil71 }7273 @Test func availabilityIsReported() async {74 let a = await Intelligence().availability()75 print("On-device model availability on this machine: \(a)")76 if case .unavailable(let r) = a { #expect(!r.explanation.isEmpty) }77 }7879 @Test(.enabled(if: RealModelTests.evaluationRequested, RealModelTests.skipComment)) func oversizedPromptMapsToContextWindowExceeded() async {80 guard Self.evaluationRequested else { return }81 guard await unavailableReason() == nil else { return }82 var big = "Region,Product,Units,Revenue\n"83 for i in 0..<3000 { big += "R\(i % 7),Widget \(i),\(i * 3 % 97),\(Double(i) * 12.5)\n" }84 do {85 _ = try await FoundationModelsBackend().respond(instructions: "Summarise the table.", prompt: big, options: GenerationOptions())86 Issue.record("an oversized prompt was accepted")87 } catch { #expect(error as? IntelligenceError == .contextWindowExceeded) }88 }8990 @Test(.enabled(if: RealModelTests.evaluationRequested, RealModelTests.skipComment)) func fullEvaluation() async throws {91 guard Self.evaluationRequested else { print("Evaluation skipped (ORIX_AI_EVAL is not 1)."); return }92 if let reason = await unavailableReason() { print("EVALUATION.md not written (\(reason))."); return }93 let backend = FoundationModelsBackend()94 let availability = await backend.availability95 let english = Intelligence(backend: backend, locale: Locale(identifier: "en_US"))96 let context = Fixtures.context(active: "H2")97 var md = "# SheetIntelligence — measured evaluation\n\n"98 md += "Generated by `ORIX_AI_EVAL=1 swift test` (`RealModelTests.fullEvaluation`) against the real on-device model. Nothing below is scripted or edited by hand. The evaluation is opt-in: a plain `swift test` (variable unset, or any value other than `1`) skips every test that calls the model and leaves this file untouched.\n\n"99 md += "- Date: \(ISO8601DateFormatter().string(from: Date()))\n- macOS: \(ProcessInfo.processInfo.operatingSystemVersionString)\n"100 md += "- Model availability: \(availability)\n- Function registry: \(FunctionRegistry.standard.count) functions (core + FunctionsNumeric + FunctionsText + FunctionsLookup)\n"101 md += "- Workbook: synthetic sales table `Sales!A1:F21` (Region, Product, Units, Unit Price, Revenue, Date), sheet `Params` with the name `TaxRate`; active cell `H2`\n- Decoding: greedy (temperature 0) for every task, streamed prose included; formulas and Python as plain text lines, review / charts / plan with guided generation\n- An earlier run of the same 20 requests with formulas generated inside guided JSON is kept verbatim in `EVALUATION-run1-guided-json.md` (13 / 20 correct); see \"Changes between runs\" at the end\n\n"102103 // 1. formula suggestions104 var rows = ""105 var suggestions = 0, valid = 0, firstPassValidRequests = 0, anyValidRequests = 0, topCorrect = 0, anyCorrect = 0, repairs = 0, repairsCorrect = 0, failures = 0106 var seconds = 0.0107 for (n, c) in Self.cases.enumerated() {108 let start = Date()109 var out: [FormulaSuggestion] = []110 var note = ""111 do { out = try await english.suggestFormulas(request: c.request, context: context) } catch { failures += 1; note = "request failed: \((error as? IntelligenceError)?.code ?? "\(error)")" }112 seconds += Date().timeIntervalSince(start)113 Self.progress("request \(n + 1)/\(Self.cases.count) in \(String(format: "%.1f", Date().timeIntervalSince(start))) s: \(out.count) suggestions \(note)")114 suggestions += out.count115 valid += out.filter(\.validation.isValid).count116 if out.contains(where: { $0.validation.isValid && !$0.isRepair }) { firstPassValidRequests += 1 }117 if out.contains(where: \.validation.isValid) { anyValidRequests += 1 }118 if let first = out.first, first.validation.isValid, Self.matches(first.validation.value, c.expected) { topCorrect += 1 }119 let correct = out.contains { $0.validation.isValid && Self.matches($0.validation.value, c.expected) }120 if correct { anyCorrect += 1 }121 if out.contains(where: \.isRepair) { repairs += 1; if out.contains(where: { $0.isRepair && $0.validation.isValid && Self.matches($0.validation.value, c.expected) }) { repairsCorrect += 1 } }122 let listing = out.map { s -> String in123 let verdict = s.validation.isValid ? (Self.matches(s.validation.value, c.expected) ? "valid, correct (\(s.validation.preview ?? ""))" : "valid, WRONG value (\(s.validation.preview ?? ""))")124 : "invalid: \(s.validation.reason ?? "")"125 return "`\(s.rawText.replacingOccurrences(of: "|", with: "\\|").replacingOccurrences(of: "\n", with: " "))`\(s.isRepair ? " (repair)" : "") — \(verdict)"126 }.joined(separator: "<br>")127 rows += "| \(n + 1) | \(c.request) | \(FormulaValidator.preview(c.expected)) | \(correct ? "yes" : "no") | \(listing.isEmpty ? note : listing) |\n"128 }129 let total = Self.cases.count130 md += "## Formula suggestions (\(total) fixed requests)\n\n"131 md += "| Measure | Result |\n|---|---|\n"132 md += "| Requests answered without error | \(total - failures) / \(total) |\n"133 md += "| Suggestions produced | \(suggestions) |\n| Suggestions that passed validation (parse, functions, references, engine evaluation) | \(valid) / \(suggestions) |\n"134 md += "| Requests with at least one valid suggestion on the first pass | \(firstPassValidRequests) / \(total) |\n"135 md += "| Requests with at least one valid suggestion (after the single repair round) | \(anyValidRequests) / \(total) |\n"136 md += "| Requests whose first listed suggestion is valid and computes the expected value | \(topCorrect) / \(total) |\n"137 md += "| Requests where some valid suggestion computes the expected value | \(anyCorrect) / \(total) |\n"138 md += "| Repair rounds triggered / repairs that produced the expected value | \(repairs) / \(repairsCorrect) |\n"139 md += "| Mean time per request | \(String(format: "%.1f", seconds / Double(total))) s |\n\n"140 md += "Expected values are computed independently in Swift from the generator of the synthetic data. A suggestion that validates but computes another value counts as wrong.\n\n"141 md += "| # | Request | Expected | Correct | Suggestions (raw model text — verdict) |\n|---|---|---|---|---|\n" + rows + "\n"142 #expect(failures < total)143144 // 2. other tasks, observed once each145 md += "## Other tasks (one observation each)\n\n"146 let (wb, engine) = Fixtures.sales()147 engine.set(sheet: 0, "H2", .formula(try FormulaParser.parse("=SUMIF(A2:A21,\"North\",E2:E21)/SUM(E2:E21)")))148 engine.set(sheet: 0, "H3", .formula(try FormulaParser.parse("=E2/(C2-C2)")))149 engine.set(sheet: 0, "E9", .formula(try FormulaParser.parse("=C9*14")))150 _ = engine.recalculate()151 func block(_ text: String) -> String { text.split(separator: "\n", omittingEmptySubsequences: false).map { "> " + $0 }.joined(separator: "\n") + "\n\n" }152 func observe(_ title: String, _ body: () async throws -> String) async {153 md += "### \(title)\n\n"154 Self.progress(title)155 do { md += try await body() } catch { md += "Failed: \((error as? IntelligenceError)?.errorDescription ?? "\(error)")\n\n" }156 }157 let formula = try #require(ContextBuilder.formulaContext(engine: engine, sheet: 0, cell: CellRef(a1: "H2")!))158 await observe("Explain a formula (English)") {159 var text = ""160 for try await s in try await english.explainFormula(formula) { text = s }161 let check = NumberVerifier.check(text, allowed: formula.steps.compactMap { Value.parseNumber($0.value) } + formula.references.compactMap { Value.parseNumber($0.value) } + [Value.parseNumber(formula.result)].compactMap { $0 }, ignoring: [formula.formula] + formula.references.map(\.address))162 #expect(!text.isEmpty)163 return "Formula `\(formula.formula)`, engine result \(formula.result).\n\n" + block(text) + "Numbers not found among the engine's values: \(check.unverified.isEmpty ? "none" : check.unverified.joined(separator: ", ")).\n\n"164 }165 let french = Intelligence(backend: backend, locale: Locale(identifier: "fr_CA"))166 await observe("Explain a formula (app locale fr-CA)") {167 let language = await french.answerLanguage()168 var text = ""169 for try await s in try await french.explainFormula(formula) { text = s }170 let words = Set(text.lowercased().split { !$0.isLetter }.map(String.init))171 let looksFrench = words.intersection(["la", "le", "les", "des", "est", "cette", "dans", "pour", "somme", "cellule", "cellules"]).count >= 3172 #expect(!text.isEmpty)173 return "`supportsLocale(fr_CA)` → answer language \(language). Answer looks French (stop-word heuristic): \(looksFrench).\n\n" + block(text)174 }175 let profile = ContextBuilder.tableProfile(workbook: wb, sheet: 0, range: RangeRef(a1: "A1:F21")!)176 await observe("Describe data") {177 let d = try await english.describeData(profile)178 return block(d.text) + "Numbers verified against the profile: \(d.verifiedNumberCount). Flagged as unverified: \(d.unverifiedNumbers.isEmpty ? "none" : d.unverifiedNumbers.joined(separator: ", ")).\n\n"179 }180 await observe("Suggest charts") {181 let charts = try await english.suggestCharts(profile)182 #expect(charts.allSatisfy { c in (c.y + [c.x].compactMap { $0 }).allSatisfy { profile.column(named: $0) != nil } })183 return charts.map { "- \($0.kind.rawValue): x = \($0.x ?? "—"), y = \($0.y.joined(separator: ", ")) — \($0.rationale)\n" }.joined() + "\n\(charts.count) accepted after validation.\n\n"184 }185 await observe("Transformation plan (goal: \"total revenue per region, highest first\")") {186 let steps = try await english.transformationPlan(goal: "total revenue per region, highest first", profile: profile)187 return steps.map { "\($0.id). \($0.operation.rawValue) [\($0.columns.joined(separator: ", "))]\($0.newColumns.isEmpty ? "" : " → [\($0.newColumns.joined(separator: ", "))]"): \($0.detail)\($0.issues.isEmpty ? "" : " — ISSUES: \($0.issues.joined(separator: " "))")\n" }.joined() + "\n\(steps.filter(\.isValid).count) of \(steps.count) steps consistent with the profile.\n\n"188 }189 let findings = ModelAuditor.audit(workbook: wb, engine: engine).findings.map(AuditFindingSummary.init)190 let audited = ContextBuilder.sheetContext(workbook: wb, sheet: 0, selection: RangeRef(a1: "H2")!, active: CellRef(a1: "H2")!)191 await observe("Review auditor findings (\(findings.count) findings from the deterministic auditor)") {192 let review = try await english.reviewForErrors(findings: findings, context: audited)193 #expect(review.items.allSatisfy { item in findings.contains { $0.id == item.findingID } })194 return block(review.overview) + review.items.map { "\($0.priority). `\($0.findingID)` — \($0.explanation) Fix: \($0.suggestedFix)\n" }.joined()195 + "\nInvented ids rejected: \(review.rejectedIDs.count).\n\n"196 }197 await observe("Explain statistics (descriptive summary of Units)") {198 let table = [["Statistic", "Units"], ["Count", "20"], ["Mean", "15.25"], ["Median", "15"], ["Std. deviation (sample)", "6.71"], ["Min", "5"], ["Max", "27"]]199 var text = ""200 for try await s in try await english.explainStatistics(table, kind: "Descriptive statistics") { text = s }201 let check = NumberVerifier.check(text, allowed: ReportRendering.numbers(table))202 return block(text) + "Numbers not in the table: \(check.unverified.isEmpty ? "none" : check.unverified.joined(separator: ", ")).\n\n"203 }204 await observe("Python help (request: \"total revenue per region written next to the table\")") {205 let p = try await english.pythonHelp(request: "Compute total revenue per region and write the result starting at cell H1", context: audited, currentCode: nil)206 return "```python\n\(p.code)\n```\n\nSyntax check (compile only, never executed): \(p.syntax). Unknown `orix` calls: \(p.unknownAPICalls.isEmpty ? "none" : p.unknownAPICalls.joined(separator: ", ")).\n\n"207 }208 await observe("Document a sheet (model paragraph only; the rest of the document is deterministic)") {209 var doc = ""210 for try await s in try await english.documentSheet(audited, audit: findings) { doc = s }211 let purpose = doc.components(separatedBy: SheetDocumenter.purposeHeading).last?.components(separatedBy: "\n## ").first ?? ""212 #expect(doc.contains("`=C2*D2`"))213 return block(purpose.trimmingCharacters(in: .whitespacesAndNewlines))214 }215 md += """216 ## Changes between runs217218 Run 1 (`EVALUATION-run1-guided-json.md`, same machine, same day, same 20 requests): 38 / 57 suggestions valid, 13 / 20 requests correct.219 Its failures showed three general defects, which were changed before this run (the 20 requests and their expected values were not changed, and no request-specific wording was added to the prompts):220221 1. Formulas were generated inside JSON strings (guided generation). Formulas contain double quotes, and the answers were cut at the first quote (`=COUNTIFS(F2:F21, `) or used a single quote instead. Formulas (and Python) are now requested as plain `FORMULA:` / `WHY:` lines and parsed strictly.222 2. The prompt listed the signatures of the functions "closest to the request" (keyword match), which pushed the model towards exotic functions (`AGGREGATE`, `CHOOSECOLS`, `EVEN`). It now lists function names only and asks for the simplest formula; it also states two general syntax facts (criteria are quoted text such as ">20"; dates are written with `DATE`).223 3. The model was asked for "up to 3 different formulas", which produced valid formulas that answer a different question. It is now asked for the best formula, with alternatives only when they answer the same request.224225 No further prompt changes were made after this run.226227 """228 let url = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("EVALUATION.md")229 try md.write(to: url, atomically: true, encoding: .utf8)230 print("EVALUATION.md written: \(anyCorrect)/\(total) requests correct, \(valid)/\(suggestions) suggestions valid.")231 }232}233