Repository Public
1// SP Notes Core2// File: Tests/SPNotesAPITests/NextAIFixtureTests.swift3// Description: The AI models decode (a) what the foundation scratch server really answers (group `ai`: the 2.x model4// catalogue, the 404 of the routes it does not have) and (b) the examples of the AI workstream's contract5// (group `ai-doc`: transcribed from API-NEXT-AI.md, NOT recorded from a server by this package).6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation10import XCTest11@testable import SPNotesAPI1213final class NextAIFixtureTests: XCTestCase {14 private func event(_ name: String, file: StaticString = #filePath, line: UInt = #line) throws -> AIEvent {15 let fixture = try NextFixture.load("ai-doc", name, file: file, line: line)16 return try XCTUnwrap(AIEvent(json: fixture.data), "\(name) is not an event", file: file, line: line)17 }1819 // MARK: recorded from the foundation server2021 func testRecordedCatalog() throws {22 let catalog = try NextFixture.decode(AIModelCatalog.self, "ai", "models")23 XCTAssertGreaterThan(catalog.models.count, 10)24 let haiku = try XCTUnwrap(catalog.models.first { $0.id == "claude-haiku-5-5" })25 XCTAssertEqual(haiku.provider, "anthropic")26 XCTAssertEqual(haiku.displayName, "Claude Haiku 5.5")27 XCTAssertEqual(haiku.category, "text")28 XCTAssertEqual(haiku.available, true)29 XCTAssertTrue(haiku.isSelectable)30 XCTAssertEqual(haiku.contextWindow, 1_000_000)31 XCTAssertEqual(haiku.maxOutput, 128_000)32 XCTAssertEqual(haiku.inputPrice, 0.1)33 XCTAssertEqual(haiku.cachedPrice, 0.01)34 XCTAssertEqual(haiku.outputPrice, 0.5)35 XCTAssertNotNil(haiku.pricingSource)36 XCTAssertTrue(try XCTUnwrap(haiku.probes).string.contains("structured_output"))37 XCTAssertTrue(try XCTUnwrap(haiku.benchmark).string.contains("pass_rate"))38 XCTAssertNotNil(haiku.capabilities)39 XCTAssertNotNil(haiku.discoveredAt)40 XCTAssertNotNil(haiku.benchmarkedAt)41 XCTAssertNotNil(haiku.updatedAt)42 // The foundation server has none of the 3.0 fields: everything stays nil instead of failing.43 XCTAssertNil(haiku.creator)44 XCTAssertNil(haiku.route)45 XCTAssertNil(haiku.priceIn)46 XCTAssertNil(haiku.autoRoutable)47 XCTAssertNil(catalog.creators)48 XCTAssertNil(catalog.hiddenUnavailable)49 XCTAssertTrue(catalog.models.allSatisfy { !$0.id.isEmpty && $0.provider != nil && $0.category != nil })5051 XCTAssertEqual(Set(catalog.providers.map(\.id)), ["openai", "anthropic"])52 let provider = try XCTUnwrap(catalog.providers.first { $0.id == "anthropic" })53 XCTAssertEqual(provider.name, "Anthropic")54 XCTAssertEqual(provider.enabled, true)55 XCTAssertNotNil(provider.configured)56 XCTAssertNil(provider.keyEnv)57 XCTAssertEqual(catalog.taskLabels?["chat"], "Chat & Q&A")58 XCTAssertTrue(try XCTUnwrap(catalog.routes).string.contains("primary"))59 XCTAssertNotNil(catalog.defaultRoutes)60 XCTAssertTrue(try XCTUnwrap(catalog.benchmark).string.contains("categories"))61 }6263 func testMissingRoutesOnAFoundationServer() throws {64 for name in ["zoom-ai-missing-route", "audio-missing-route", "database-ai-config-missing-route", "database-ai-runs-missing-route"] {65 let fixture = try NextFixture.load("ai", name)66 XCTAssertEqual(fixture.entry.status, 404, name)67 XCTAssertTrue(fixture.entry.headers["content-type"]?.hasPrefix("text/html") ?? false, "the HTML not-found page, not a JSON error")68 let error = try NextFixture.error("ai", name)69 XCTAssertTrue(error.isNotFound)70 XCTAssertNil(error.code)71 XCTAssertTrue(error.isRouteMissing, "\(name): a non-JSON 404 means the route does not exist on this server")72 }73 }7475 // MARK: transcribed from the AI workstream's document7677 func testDocumentCatalogFields() throws {78 let model = try NextFixture.decode(AIModelInfo.self, "ai-doc", "model-openrouter")79 XCTAssertEqual(model.id, "openrouter:anthropic/claude-fable-5")80 XCTAssertEqual(model.provider, "openrouter")81 XCTAssertEqual(model.creator, "anthropic")82 XCTAssertEqual(model.creatorName, "Anthropic")83 XCTAssertEqual(model.logo, "/model-logos/anthropic.svg")84 XCTAssertEqual(model.monogram, "AN")85 XCTAssertEqual(model.route, "openrouter")86 XCTAssertEqual(model.routeLabel, "OpenRouter")87 XCTAssertEqual(model.routeKind, AIRouteKind.openAICompatible)88 XCTAssertEqual(model.routeLogo, "/model-logos/openrouter.svg")89 XCTAssertEqual(model.remoteId, "anthropic/claude-fable-5")90 XCTAssertEqual(model.context, 1_000_000)91 XCTAssertEqual(model.context, model.contextWindow)92 XCTAssertEqual(model.priceIn, 10)93 XCTAssertEqual(model.priceOut, 50)94 XCTAssertEqual(model.priceCached, 1)95 XCTAssertEqual(model.priceUnit, "USD per 1M tokens")96 XCTAssertEqual(model.listed, true)97 XCTAssertEqual(model.available, false)98 XCTAssertFalse(model.isSelectable)99 XCTAssertEqual(model.unavailableReason, AIUnavailableReason.noKey)100 XCTAssertEqual(model.benchmarked, false)101 XCTAssertEqual(model.autoRoutable, false)102 XCTAssertEqual(model.notes?.count, 1)103 let capabilities = try XCTUnwrap(model.normalizedCapabilities)104 XCTAssertEqual(capabilities, AIModelCapabilities(text: true, streaming: true, tools: true, structuredOutput: true, jsonMode: true, vision: true, reasoning: true, source: "declared"))105106 let providers = try NextFixture.decode([AIProviderInfo].self, "ai-doc", "models-providers")107 XCTAssertEqual(providers.map(\.id), ["openai", "anthropic", "openrouter", "maclustr"])108 let openrouter = providers[2]109 XCTAssertEqual(openrouter.kind, AIRouteKind.openAICompatible)110 XCTAssertEqual(openrouter.baseUrl, "https://openrouter.ai/api/v1")111 XCTAssertEqual(openrouter.keyEnv, "OPENROUTER_API_KEY", "the name of the variable, never a key")112 XCTAssertEqual(openrouter.label, "OpenRouter")113 XCTAssertEqual(openrouter.configured, false)114 XCTAssertEqual(openrouter.available, false)115 XCTAssertEqual(openrouter.status, "no_key")116 XCTAssertNotNil(openrouter.statusDetail)117 XCTAssertNotNil(openrouter.lastDiscoveryAt)118 XCTAssertEqual(openrouter.logo, "/model-logos/openrouter.svg")119 XCTAssertEqual(providers[1].kind, AIRouteKind.anthropic)120 XCTAssertEqual(providers[1].available, true)121 XCTAssertNil(providers[0].lastDiscoveryAt)122123 let creators = try NextFixture.decode([AIModelCreator].self, "ai-doc", "models-creators-first-two")124 XCTAssertEqual(creators, [AIModelCreator(id: "anthropic", name: "Anthropic", logo: "/model-logos/anthropic.svg", monogram: "AN", generic: false),125 AIModelCreator(id: "openai", name: "OpenAI", logo: "/model-logos/generic.svg", monogram: "OA", generic: true)])126 }127128 func testDocumentChatScopeAndSources() throws {129 guard case .meta(let meta) = try event("chat-page-meta") else { return XCTFail("meta expected") }130 XCTAssertEqual(meta.model, "claude-haiku-5-5")131 XCTAssertEqual(meta.conversationId, "fe873d65-75c5-4827-89d2-37e1e48918f6")132 XCTAssertEqual(meta.scope, AIChatScope.page)133 XCTAssertEqual(meta.scopePages, 1)134 XCTAssertEqual(meta.staleDropped, 0)135 XCTAssertEqual(meta.evidence, "ok")136 XCTAssertEqual(meta.mode, "notes")137 XCTAssertEqual(meta.task, "chat")138 XCTAssertEqual(meta.clientId, "live-ask-page-02")139 XCTAssertEqual(meta.replayed, false)140 XCTAssertTrue(meta.weakSources.isEmpty)141 let source = try XCTUnwrap(meta.sources.first)142 XCTAssertEqual(source.n, 1)143 XCTAssertEqual(source.chunkId, "a8a64c4b-bafb-4afe-b4a6-2594f911b667")144 XCTAssertEqual(source.pageId, "2eea0ded-b5b8-4abe-b560-75345504be80")145 XCTAssertEqual(source.title, "Offre de la banque")146 XCTAssertEqual(source.heading, "Conditions")147 XCTAssertEqual(source.blockId, "XycxLnbyuq", "the block that best supports the citation")148 XCTAssertEqual(source.blockIds, ["RttMYFvegP", "XycxLnbyuq", "8As7edcAjg"])149 XCTAssertEqual(source.anchor, AISourceAnchor.block)150 XCTAssertEqual(source.quote, "La pénalité de remboursement anticipé équivaut à trois mois d intérêts.")151 XCTAssertEqual(source.sourceType, "page")152 XCTAssertEqual(source.semantic, 0.644)153 XCTAssertNotNil(source.score)154 XCTAssertNil(source.icon)155156 guard case .done(let done) = try event("chat-page-done") else { return XCTFail("done expected") }157 XCTAssertEqual(done.messageId, "e52caff5-2aa9-4bea-aafe-46431bc80021")158 XCTAssertEqual(done.invalidCitations, [])159 XCTAssertEqual(done.uncited, false)160 XCTAssertEqual(done.evidence, "ok")161 XCTAssertEqual(done.replayed, false)162 XCTAssertEqual(done.ttftMs, 953)163 XCTAssertEqual(done.latencyMs, 1213)164 XCTAssertEqual(done.sources.first?.anchor, AISourceAnchor.block)165 XCTAssertEqual(done.sources.first?.blockIds?.count, 3)166 XCTAssertTrue(done.content?.contains("[1]") ?? false)167168 // No supporting source: no provider was called.169 guard case .meta(let none) = try event("chat-none-meta") else { return XCTFail("meta expected") }170 XCTAssertNil(none.model)171 XCTAssertEqual(none.evidence, "none")172 XCTAssertEqual(none.scope, AIChatScope.project)173 XCTAssertEqual(none.scopePages, 2)174 XCTAssertTrue(none.sources.isEmpty)175 guard case .done(let noneDone) = try event("chat-none-done") else { return XCTFail("done expected") }176 XCTAssertEqual(noneDone.costUsd, 0)177 XCTAssertNil(noneDone.usage)178 XCTAssertNil(noneDone.ttftMs)179 XCTAssertEqual(noneDone.uncited, false)180 XCTAssertTrue(noneDone.content?.hasPrefix("**No supporting note found.**") ?? false)181182 guard case .error(let error) = try event("chat-error-page-required") else { return XCTFail("error expected") }183 XCTAssertEqual(error.code, APIError.AICode.pageRequired)184 XCTAssertFalse(error.retryable)185186 // A 2.x event (no 3.0 key) reads as nil / empty, never a failure.187 guard case .meta(let legacy)? = AIEvent(json: Data(#"{"type":"meta","model":"m","sources":[{"n":1,"pageId":"p","title":"T"}]}"#.utf8)) else { return XCTFail("meta expected") }188 XCTAssertNil(legacy.scope)189 XCTAssertNil(legacy.scopePages)190 XCTAssertNil(legacy.staleDropped)191 XCTAssertEqual(legacy.sources.first?.title, "T")192 XCTAssertNil(legacy.sources.first?.blockIds)193 XCTAssertNil(legacy.sources.first?.anchor)194 }195196 func testDocumentLinkSuggestions() throws {197 guard case .meta(let meta) = try event("action-meta") else { return XCTFail("meta expected") }198 XCTAssertEqual(meta.provider, "anthropic")199 XCTAssertNil(meta.scope)200201 let result = try event("action-links-result")202 guard case .other(let type, _) = result else { return XCTFail("a result event is delivered as .other") }203 XCTAssertEqual(type, AIEvent.resultType)204 let suggestions = try XCTUnwrap(result.linkSuggestions)205 XCTAssertEqual(suggestions.ai, true)206 XCTAssertEqual(suggestions.model, "claude-haiku-5-5")207 XCTAssertEqual(suggestions.candidates, 4)208 XCTAssertEqual(suggestions.links.count, 2)209 let first = suggestions.links[0]210 XCTAssertEqual(first.pageId, "2eea0ded-b5b8-4abe-b560-75345504be80")211 XCTAssertEqual(first.title, "Offre de la banque")212 XCTAssertEqual(first.blockId, "RttMYFvegP")213 XCTAssertEqual(first.anchorText, "pénalité de remboursement anticipé")214 XCTAssertTrue(first.reason?.hasPrefix("Donne les conditions") ?? false)215 XCTAssertTrue(first.excerpt?.contains("4,29 %") ?? false)216 XCTAssertEqual(suggestions.links[1].score, 0.015625)217 XCTAssertNil(try event("action-meta").linkSuggestions, "only a result event carries suggestions")218219 guard case .done(let done) = try event("action-links-done") else { return XCTFail("done expected") }220 XCTAssertEqual(done.latencyMs, 2269)221 XCTAssertNotNil(done.usage)222 XCTAssertNil(done.invalidCitations)223224 XCTAssertEqual(AIActionID.all.count, 26)225 XCTAssertEqual(Set(AIActionID.all).count, 26)226 }227228 func testDocumentSynopsis() throws {229 let ready = try NextFixture.decode(ZoomSynopsis.self, "ai-doc", "zoom-ai-ready")230 XCTAssertEqual(ready.pageId, "2eea0ded-b5b8-4abe-b560-75345504be80")231 XCTAssertEqual(ready.level, 1)232 XCTAssertEqual(ready.contentHash, "20156a7accd058b5857a3744a185c6de")233 XCTAssertTrue(ready.ai)234 XCTAssertEqual(ready.status, SynopsisStatus.ready)235 XCTAssertTrue(ready.synopsis?.hasPrefix("La note décrit une offre de prêt bancaire") ?? false)236 XCTAssertEqual(ready.model, "claude-haiku-5-5")237 XCTAssertEqual(ready.provider, "anthropic")238 XCTAssertEqual(ready.generatedAt.map(APIDate.string(from:)), "2026-11-19T18:22:22.977Z")239 XCTAssertEqual(ready.cached, false)240 XCTAssertEqual(ready.costUsd, 0.0001022)241242 let missing = try NextFixture.decode(ZoomSynopsis.self, "ai-doc", "zoom-ai-missing")243 XCTAssertFalse(missing.ai)244 XCTAssertEqual(missing.status, SynopsisStatus.missing)245 XCTAssertNil(missing.synopsis)246 XCTAssertNil(missing.generatedAt)247 XCTAssertNil(missing.costUsd)248249 // `GET …/zoom?level=1` returns the same object: PageZoom exposes it as `synopsis`.250 let zoom = try NextFixture.decode(PageZoom.self, "ai-doc", "zoom-ai-ready")251 XCTAssertEqual(zoom.level, 1)252 XCTAssertTrue(zoom.ai)253 XCTAssertEqual(zoom.synopsis, ready)254 XCTAssertNil(zoom.sections)255 XCTAssertNil(zoom.title)256 }257258 func testDocumentTranscription() throws {259 let queued = try NextFixture.decode(AudioTranscription.self, "ai-doc", "audio-transcribe-queued")260 XCTAssertEqual(try NextFixture.load("ai-doc", "audio-transcribe-queued").entry.status, 202)261 XCTAssertEqual(queued.queued, true)262 XCTAssertEqual(queued.jobId, "1b60aa7a-da87-4306-b323-e37c5f233ca9")263 XCTAssertEqual(queued.status, TranscriptionStatus.pending)264 XCTAssertTrue(queued.isPending)265 XCTAssertEqual(queued.transcript, [])266 XCTAssertEqual(queued.text, "")267 XCTAssertNil(queued.engine)268 XCTAssertNil(queued.segments)269 XCTAssertNil(queued.summary)270 XCTAssertNotNil(queued.requestedAt)271 XCTAssertNil(queued.finishedAt)272 XCTAssertNil(queued.engineLabel)273274 let done = try NextFixture.decode(AudioTranscription.self, "ai-doc", "audio-done")275 XCTAssertEqual(done.attachmentId, queued.attachmentId)276 XCTAssertTrue(done.isDone)277 XCTAssertNil(done.queued)278 XCTAssertEqual(done.transcript, [AudioTranscriptSegment(t: 0, text: "Réunion du 19 novembre. Nous retenons le taux fixe de 5 ans."),279 AudioTranscriptSegment(t: 4.4, text: "Marie envoie les documents à la banque lundi matin.")])280 XCTAssertEqual(done.engine, "openai")281 XCTAssertEqual(done.model, "whisper-1")282 XCTAssertEqual(done.engineLabel, "openai:whisper-1")283 XCTAssertEqual(done.language, "french")284 XCTAssertEqual(try XCTUnwrap(done.duration), 6.87, accuracy: 0.001)285 XCTAssertEqual(done.segments, true)286 let summary = try XCTUnwrap(done.summary)287 XCTAssertEqual(summary.model, "gpt-5.4-mini")288 XCTAssertEqual(summary.tasks, ["Marie envoie les documents à la banque lundi matin."])289 XCTAssertEqual(summary.decisions, ["Ils retiennent le taux fixe de 5 ans."])290 XCTAssertNotNil(summary.at)291 XCTAssertNil(done.error)292 XCTAssertNil(done.costUsd)293 XCTAssertNotNil(done.finishedAt)294 }295296 func testDocumentSmartDatabaseProperties() throws {297 let config = try NextFixture.decode(DatabaseAIConfig.self, "ai-doc", "database-ai-config")298 XCTAssertEqual(config.tasks, [DatabaseAITask.custom, DatabaseAITask.summarize, DatabaseAITask.classify, DatabaseAITask.extract, DatabaseAITask.detectDate, DatabaseAITask.categorize])299 let property = try XCTUnwrap(config.properties.first)300 XCTAssertEqual(property.propertyId, "3981f845-97b9-47f0-b153-96ef517c8e4c")301 XCTAssertEqual(property.name, "Priorité IA")302 XCTAssertEqual(property.ai, DatabaseAISettings(task: DatabaseAITask.classify, prompt: "", choices: ["Urgent", "Normal", "Plus tard"], model: "claude-haiku-5-5", autoRun: true, managed: false))303 XCTAssertNil(property.problem)304 let runs = try XCTUnwrap(property.runs)305 XCTAssertEqual(runs.done, 1)306 XCTAssertEqual(runs.skipped, 1)307 XCTAssertEqual(runs.costUsd, 0.0000626)308 XCTAssertNotNil(runs.lastAt)309310 let saved = try NextFixture.decode(DatabaseAIProperty.self, "ai-doc", "database-ai-config-put")311 XCTAssertEqual(saved.ai, property.ai)312 XCTAssertNil(saved.runs, "the PUT answer has no counters")313314 let run = try NextFixture.decode(DatabaseAIRunResult.self, "ai-doc", "database-ai-run-skipped")315 XCTAssertEqual(run.ok, true)316 XCTAssertEqual(run.queued, 0)317 XCTAssertEqual(run.runs, [DatabaseAIRowRun(rowId: "93e5f9e5-5f6e-4c40-ab8c-9ac28e28c524", runId: "d93a25ed-1f05-4353-ab79-f8f1e60082be", queued: false, skipped: DatabaseAISkipReason.userEdited)])318 XCTAssertEqual(run.skipped?.first?.reason, DatabaseAISkipReason.userEdited)319320 let log = try NextFixture.decode([DatabaseAIRun].self, "ai-doc", "database-ai-runs")321 let entry = try XCTUnwrap(log.first)322 XCTAssertEqual(entry.status, "done")323 XCTAssertEqual(entry.trigger, "auto")324 XCTAssertEqual(entry.task, DatabaseAITask.classify)325 XCTAssertEqual(entry.inputTokens, 561)326 XCTAssertEqual(entry.outputTokens, 13)327 XCTAssertEqual(entry.costUsd, 0.0000626)328 XCTAssertNil(entry.error)329 XCTAssertLessThan(try XCTUnwrap(entry.startedAt), try XCTUnwrap(entry.finishedAt))330 }331332 func testEveryDocumentFixtureIsUsed() throws {333 // Guards against a transcribed example nobody decodes.334 let used: Set<String> = ["models-providers", "models-creators-first-two", "model-openrouter", "chat-page-meta", "chat-page-done", "chat-none-meta",335 "chat-none-done", "chat-error-page-required", "action-meta", "action-links-result", "action-links-done", "zoom-ai-ready",336 "zoom-ai-missing", "audio-transcribe-queued", "audio-done", "database-ai-config", "database-ai-config-put",337 "database-ai-run-skipped", "database-ai-runs"]338 XCTAssertEqual(Set(try NextFixture.index("ai-doc").map(\.name)), used)339 }340}341