SPB Git

Repository Public

NextAIFixtureTests.swift 18.7 KB 341 lines swift

1// SP Notes Core2// File: Tests/SPNotesAPITests/NextAIFixtureTests.swift3// Description: The AI models decode (a) what the foundation scratch server really answers (group `ai`: the 2.x model4//              catalogue, the 404 of the routes it does not have) and (b) the examples of the AI workstream's contract5//              (group `ai-doc`: transcribed from API-NEXT-AI.md, NOT recorded from a server by this package).6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation10import XCTest11@testable import SPNotesAPI1213final class NextAIFixtureTests: XCTestCase {14    private func event(_ name: String, file: StaticString = #filePath, line: UInt = #line) throws -> AIEvent {15        let fixture = try NextFixture.load("ai-doc", name, file: file, line: line)16        return try XCTUnwrap(AIEvent(json: fixture.data), "\(name) is not an event", file: file, line: line)17    }1819    // MARK: recorded from the foundation server2021    func testRecordedCatalog() throws {22        let catalog = try NextFixture.decode(AIModelCatalog.self, "ai", "models")23        XCTAssertGreaterThan(catalog.models.count, 10)24        let haiku = try XCTUnwrap(catalog.models.first { $0.id == "claude-haiku-5-5" })25        XCTAssertEqual(haiku.provider, "anthropic")26        XCTAssertEqual(haiku.displayName, "Claude Haiku 5.5")27        XCTAssertEqual(haiku.category, "text")28        XCTAssertEqual(haiku.available, true)29        XCTAssertTrue(haiku.isSelectable)30        XCTAssertEqual(haiku.contextWindow, 1_000_000)31        XCTAssertEqual(haiku.maxOutput, 128_000)32        XCTAssertEqual(haiku.inputPrice, 0.1)33        XCTAssertEqual(haiku.cachedPrice, 0.01)34        XCTAssertEqual(haiku.outputPrice, 0.5)35        XCTAssertNotNil(haiku.pricingSource)36        XCTAssertTrue(try XCTUnwrap(haiku.probes).string.contains("structured_output"))37        XCTAssertTrue(try XCTUnwrap(haiku.benchmark).string.contains("pass_rate"))38        XCTAssertNotNil(haiku.capabilities)39        XCTAssertNotNil(haiku.discoveredAt)40        XCTAssertNotNil(haiku.benchmarkedAt)41        XCTAssertNotNil(haiku.updatedAt)42        // The foundation server has none of the 3.0 fields: everything stays nil instead of failing.43        XCTAssertNil(haiku.creator)44        XCTAssertNil(haiku.route)45        XCTAssertNil(haiku.priceIn)46        XCTAssertNil(haiku.autoRoutable)47        XCTAssertNil(catalog.creators)48        XCTAssertNil(catalog.hiddenUnavailable)49        XCTAssertTrue(catalog.models.allSatisfy { !$0.id.isEmpty && $0.provider != nil && $0.category != nil })5051        XCTAssertEqual(Set(catalog.providers.map(\.id)), ["openai", "anthropic"])52        let provider = try XCTUnwrap(catalog.providers.first { $0.id == "anthropic" })53        XCTAssertEqual(provider.name, "Anthropic")54        XCTAssertEqual(provider.enabled, true)55        XCTAssertNotNil(provider.configured)56        XCTAssertNil(provider.keyEnv)57        XCTAssertEqual(catalog.taskLabels?["chat"], "Chat & Q&A")58        XCTAssertTrue(try XCTUnwrap(catalog.routes).string.contains("primary"))59        XCTAssertNotNil(catalog.defaultRoutes)60        XCTAssertTrue(try XCTUnwrap(catalog.benchmark).string.contains("categories"))61    }6263    func testMissingRoutesOnAFoundationServer() throws {64        for name in ["zoom-ai-missing-route", "audio-missing-route", "database-ai-config-missing-route", "database-ai-runs-missing-route"] {65            let fixture = try NextFixture.load("ai", name)66            XCTAssertEqual(fixture.entry.status, 404, name)67            XCTAssertTrue(fixture.entry.headers["content-type"]?.hasPrefix("text/html") ?? false, "the HTML not-found page, not a JSON error")68            let error = try NextFixture.error("ai", name)69            XCTAssertTrue(error.isNotFound)70            XCTAssertNil(error.code)71            XCTAssertTrue(error.isRouteMissing, "\(name): a non-JSON 404 means the route does not exist on this server")72        }73    }7475    // MARK: transcribed from the AI workstream's document7677    func testDocumentCatalogFields() throws {78        let model = try NextFixture.decode(AIModelInfo.self, "ai-doc", "model-openrouter")79        XCTAssertEqual(model.id, "openrouter:anthropic/claude-fable-5")80        XCTAssertEqual(model.provider, "openrouter")81        XCTAssertEqual(model.creator, "anthropic")82        XCTAssertEqual(model.creatorName, "Anthropic")83        XCTAssertEqual(model.logo, "/model-logos/anthropic.svg")84        XCTAssertEqual(model.monogram, "AN")85        XCTAssertEqual(model.route, "openrouter")86        XCTAssertEqual(model.routeLabel, "OpenRouter")87        XCTAssertEqual(model.routeKind, AIRouteKind.openAICompatible)88        XCTAssertEqual(model.routeLogo, "/model-logos/openrouter.svg")89        XCTAssertEqual(model.remoteId, "anthropic/claude-fable-5")90        XCTAssertEqual(model.context, 1_000_000)91        XCTAssertEqual(model.context, model.contextWindow)92        XCTAssertEqual(model.priceIn, 10)93        XCTAssertEqual(model.priceOut, 50)94        XCTAssertEqual(model.priceCached, 1)95        XCTAssertEqual(model.priceUnit, "USD per 1M tokens")96        XCTAssertEqual(model.listed, true)97        XCTAssertEqual(model.available, false)98        XCTAssertFalse(model.isSelectable)99        XCTAssertEqual(model.unavailableReason, AIUnavailableReason.noKey)100        XCTAssertEqual(model.benchmarked, false)101        XCTAssertEqual(model.autoRoutable, false)102        XCTAssertEqual(model.notes?.count, 1)103        let capabilities = try XCTUnwrap(model.normalizedCapabilities)104        XCTAssertEqual(capabilities, AIModelCapabilities(text: true, streaming: true, tools: true, structuredOutput: true, jsonMode: true, vision: true, reasoning: true, source: "declared"))105106        let providers = try NextFixture.decode([AIProviderInfo].self, "ai-doc", "models-providers")107        XCTAssertEqual(providers.map(\.id), ["openai", "anthropic", "openrouter", "maclustr"])108        let openrouter = providers[2]109        XCTAssertEqual(openrouter.kind, AIRouteKind.openAICompatible)110        XCTAssertEqual(openrouter.baseUrl, "https://openrouter.ai/api/v1")111        XCTAssertEqual(openrouter.keyEnv, "OPENROUTER_API_KEY", "the name of the variable, never a key")112        XCTAssertEqual(openrouter.label, "OpenRouter")113        XCTAssertEqual(openrouter.configured, false)114        XCTAssertEqual(openrouter.available, false)115        XCTAssertEqual(openrouter.status, "no_key")116        XCTAssertNotNil(openrouter.statusDetail)117        XCTAssertNotNil(openrouter.lastDiscoveryAt)118        XCTAssertEqual(openrouter.logo, "/model-logos/openrouter.svg")119        XCTAssertEqual(providers[1].kind, AIRouteKind.anthropic)120        XCTAssertEqual(providers[1].available, true)121        XCTAssertNil(providers[0].lastDiscoveryAt)122123        let creators = try NextFixture.decode([AIModelCreator].self, "ai-doc", "models-creators-first-two")124        XCTAssertEqual(creators, [AIModelCreator(id: "anthropic", name: "Anthropic", logo: "/model-logos/anthropic.svg", monogram: "AN", generic: false),125                                  AIModelCreator(id: "openai", name: "OpenAI", logo: "/model-logos/generic.svg", monogram: "OA", generic: true)])126    }127128    func testDocumentChatScopeAndSources() throws {129        guard case .meta(let meta) = try event("chat-page-meta") else { return XCTFail("meta expected") }130        XCTAssertEqual(meta.model, "claude-haiku-5-5")131        XCTAssertEqual(meta.conversationId, "fe873d65-75c5-4827-89d2-37e1e48918f6")132        XCTAssertEqual(meta.scope, AIChatScope.page)133        XCTAssertEqual(meta.scopePages, 1)134        XCTAssertEqual(meta.staleDropped, 0)135        XCTAssertEqual(meta.evidence, "ok")136        XCTAssertEqual(meta.mode, "notes")137        XCTAssertEqual(meta.task, "chat")138        XCTAssertEqual(meta.clientId, "live-ask-page-02")139        XCTAssertEqual(meta.replayed, false)140        XCTAssertTrue(meta.weakSources.isEmpty)141        let source = try XCTUnwrap(meta.sources.first)142        XCTAssertEqual(source.n, 1)143        XCTAssertEqual(source.chunkId, "a8a64c4b-bafb-4afe-b4a6-2594f911b667")144        XCTAssertEqual(source.pageId, "2eea0ded-b5b8-4abe-b560-75345504be80")145        XCTAssertEqual(source.title, "Offre de la banque")146        XCTAssertEqual(source.heading, "Conditions")147        XCTAssertEqual(source.blockId, "XycxLnbyuq", "the block that best supports the citation")148        XCTAssertEqual(source.blockIds, ["RttMYFvegP", "XycxLnbyuq", "8As7edcAjg"])149        XCTAssertEqual(source.anchor, AISourceAnchor.block)150        XCTAssertEqual(source.quote, "La pénalité de remboursement anticipé équivaut à trois mois d intérêts.")151        XCTAssertEqual(source.sourceType, "page")152        XCTAssertEqual(source.semantic, 0.644)153        XCTAssertNotNil(source.score)154        XCTAssertNil(source.icon)155156        guard case .done(let done) = try event("chat-page-done") else { return XCTFail("done expected") }157        XCTAssertEqual(done.messageId, "e52caff5-2aa9-4bea-aafe-46431bc80021")158        XCTAssertEqual(done.invalidCitations, [])159        XCTAssertEqual(done.uncited, false)160        XCTAssertEqual(done.evidence, "ok")161        XCTAssertEqual(done.replayed, false)162        XCTAssertEqual(done.ttftMs, 953)163        XCTAssertEqual(done.latencyMs, 1213)164        XCTAssertEqual(done.sources.first?.anchor, AISourceAnchor.block)165        XCTAssertEqual(done.sources.first?.blockIds?.count, 3)166        XCTAssertTrue(done.content?.contains("[1]") ?? false)167168        // No supporting source: no provider was called.169        guard case .meta(let none) = try event("chat-none-meta") else { return XCTFail("meta expected") }170        XCTAssertNil(none.model)171        XCTAssertEqual(none.evidence, "none")172        XCTAssertEqual(none.scope, AIChatScope.project)173        XCTAssertEqual(none.scopePages, 2)174        XCTAssertTrue(none.sources.isEmpty)175        guard case .done(let noneDone) = try event("chat-none-done") else { return XCTFail("done expected") }176        XCTAssertEqual(noneDone.costUsd, 0)177        XCTAssertNil(noneDone.usage)178        XCTAssertNil(noneDone.ttftMs)179        XCTAssertEqual(noneDone.uncited, false)180        XCTAssertTrue(noneDone.content?.hasPrefix("**No supporting note found.**") ?? false)181182        guard case .error(let error) = try event("chat-error-page-required") else { return XCTFail("error expected") }183        XCTAssertEqual(error.code, APIError.AICode.pageRequired)184        XCTAssertFalse(error.retryable)185186        // A 2.x event (no 3.0 key) reads as nil / empty, never a failure.187        guard case .meta(let legacy)? = AIEvent(json: Data(#"{"type":"meta","model":"m","sources":[{"n":1,"pageId":"p","title":"T"}]}"#.utf8)) else { return XCTFail("meta expected") }188        XCTAssertNil(legacy.scope)189        XCTAssertNil(legacy.scopePages)190        XCTAssertNil(legacy.staleDropped)191        XCTAssertEqual(legacy.sources.first?.title, "T")192        XCTAssertNil(legacy.sources.first?.blockIds)193        XCTAssertNil(legacy.sources.first?.anchor)194    }195196    func testDocumentLinkSuggestions() throws {197        guard case .meta(let meta) = try event("action-meta") else { return XCTFail("meta expected") }198        XCTAssertEqual(meta.provider, "anthropic")199        XCTAssertNil(meta.scope)200201        let result = try event("action-links-result")202        guard case .other(let type, _) = result else { return XCTFail("a result event is delivered as .other") }203        XCTAssertEqual(type, AIEvent.resultType)204        let suggestions = try XCTUnwrap(result.linkSuggestions)205        XCTAssertEqual(suggestions.ai, true)206        XCTAssertEqual(suggestions.model, "claude-haiku-5-5")207        XCTAssertEqual(suggestions.candidates, 4)208        XCTAssertEqual(suggestions.links.count, 2)209        let first = suggestions.links[0]210        XCTAssertEqual(first.pageId, "2eea0ded-b5b8-4abe-b560-75345504be80")211        XCTAssertEqual(first.title, "Offre de la banque")212        XCTAssertEqual(first.blockId, "RttMYFvegP")213        XCTAssertEqual(first.anchorText, "pénalité de remboursement anticipé")214        XCTAssertTrue(first.reason?.hasPrefix("Donne les conditions") ?? false)215        XCTAssertTrue(first.excerpt?.contains("4,29 %") ?? false)216        XCTAssertEqual(suggestions.links[1].score, 0.015625)217        XCTAssertNil(try event("action-meta").linkSuggestions, "only a result event carries suggestions")218219        guard case .done(let done) = try event("action-links-done") else { return XCTFail("done expected") }220        XCTAssertEqual(done.latencyMs, 2269)221        XCTAssertNotNil(done.usage)222        XCTAssertNil(done.invalidCitations)223224        XCTAssertEqual(AIActionID.all.count, 26)225        XCTAssertEqual(Set(AIActionID.all).count, 26)226    }227228    func testDocumentSynopsis() throws {229        let ready = try NextFixture.decode(ZoomSynopsis.self, "ai-doc", "zoom-ai-ready")230        XCTAssertEqual(ready.pageId, "2eea0ded-b5b8-4abe-b560-75345504be80")231        XCTAssertEqual(ready.level, 1)232        XCTAssertEqual(ready.contentHash, "20156a7accd058b5857a3744a185c6de")233        XCTAssertTrue(ready.ai)234        XCTAssertEqual(ready.status, SynopsisStatus.ready)235        XCTAssertTrue(ready.synopsis?.hasPrefix("La note décrit une offre de prêt bancaire") ?? false)236        XCTAssertEqual(ready.model, "claude-haiku-5-5")237        XCTAssertEqual(ready.provider, "anthropic")238        XCTAssertEqual(ready.generatedAt.map(APIDate.string(from:)), "2026-11-19T18:22:22.977Z")239        XCTAssertEqual(ready.cached, false)240        XCTAssertEqual(ready.costUsd, 0.0001022)241242        let missing = try NextFixture.decode(ZoomSynopsis.self, "ai-doc", "zoom-ai-missing")243        XCTAssertFalse(missing.ai)244        XCTAssertEqual(missing.status, SynopsisStatus.missing)245        XCTAssertNil(missing.synopsis)246        XCTAssertNil(missing.generatedAt)247        XCTAssertNil(missing.costUsd)248249        // `GET …/zoom?level=1` returns the same object: PageZoom exposes it as `synopsis`.250        let zoom = try NextFixture.decode(PageZoom.self, "ai-doc", "zoom-ai-ready")251        XCTAssertEqual(zoom.level, 1)252        XCTAssertTrue(zoom.ai)253        XCTAssertEqual(zoom.synopsis, ready)254        XCTAssertNil(zoom.sections)255        XCTAssertNil(zoom.title)256    }257258    func testDocumentTranscription() throws {259        let queued = try NextFixture.decode(AudioTranscription.self, "ai-doc", "audio-transcribe-queued")260        XCTAssertEqual(try NextFixture.load("ai-doc", "audio-transcribe-queued").entry.status, 202)261        XCTAssertEqual(queued.queued, true)262        XCTAssertEqual(queued.jobId, "1b60aa7a-da87-4306-b323-e37c5f233ca9")263        XCTAssertEqual(queued.status, TranscriptionStatus.pending)264        XCTAssertTrue(queued.isPending)265        XCTAssertEqual(queued.transcript, [])266        XCTAssertEqual(queued.text, "")267        XCTAssertNil(queued.engine)268        XCTAssertNil(queued.segments)269        XCTAssertNil(queued.summary)270        XCTAssertNotNil(queued.requestedAt)271        XCTAssertNil(queued.finishedAt)272        XCTAssertNil(queued.engineLabel)273274        let done = try NextFixture.decode(AudioTranscription.self, "ai-doc", "audio-done")275        XCTAssertEqual(done.attachmentId, queued.attachmentId)276        XCTAssertTrue(done.isDone)277        XCTAssertNil(done.queued)278        XCTAssertEqual(done.transcript, [AudioTranscriptSegment(t: 0, text: "Réunion du 19 novembre. Nous retenons le taux fixe de 5 ans."),279                                         AudioTranscriptSegment(t: 4.4, text: "Marie envoie les documents à la banque lundi matin.")])280        XCTAssertEqual(done.engine, "openai")281        XCTAssertEqual(done.model, "whisper-1")282        XCTAssertEqual(done.engineLabel, "openai:whisper-1")283        XCTAssertEqual(done.language, "french")284        XCTAssertEqual(try XCTUnwrap(done.duration), 6.87, accuracy: 0.001)285        XCTAssertEqual(done.segments, true)286        let summary = try XCTUnwrap(done.summary)287        XCTAssertEqual(summary.model, "gpt-5.4-mini")288        XCTAssertEqual(summary.tasks, ["Marie envoie les documents à la banque lundi matin."])289        XCTAssertEqual(summary.decisions, ["Ils retiennent le taux fixe de 5 ans."])290        XCTAssertNotNil(summary.at)291        XCTAssertNil(done.error)292        XCTAssertNil(done.costUsd)293        XCTAssertNotNil(done.finishedAt)294    }295296    func testDocumentSmartDatabaseProperties() throws {297        let config = try NextFixture.decode(DatabaseAIConfig.self, "ai-doc", "database-ai-config")298        XCTAssertEqual(config.tasks, [DatabaseAITask.custom, DatabaseAITask.summarize, DatabaseAITask.classify, DatabaseAITask.extract, DatabaseAITask.detectDate, DatabaseAITask.categorize])299        let property = try XCTUnwrap(config.properties.first)300        XCTAssertEqual(property.propertyId, "3981f845-97b9-47f0-b153-96ef517c8e4c")301        XCTAssertEqual(property.name, "Priorité IA")302        XCTAssertEqual(property.ai, DatabaseAISettings(task: DatabaseAITask.classify, prompt: "", choices: ["Urgent", "Normal", "Plus tard"], model: "claude-haiku-5-5", autoRun: true, managed: false))303        XCTAssertNil(property.problem)304        let runs = try XCTUnwrap(property.runs)305        XCTAssertEqual(runs.done, 1)306        XCTAssertEqual(runs.skipped, 1)307        XCTAssertEqual(runs.costUsd, 0.0000626)308        XCTAssertNotNil(runs.lastAt)309310        let saved = try NextFixture.decode(DatabaseAIProperty.self, "ai-doc", "database-ai-config-put")311        XCTAssertEqual(saved.ai, property.ai)312        XCTAssertNil(saved.runs, "the PUT answer has no counters")313314        let run = try NextFixture.decode(DatabaseAIRunResult.self, "ai-doc", "database-ai-run-skipped")315        XCTAssertEqual(run.ok, true)316        XCTAssertEqual(run.queued, 0)317        XCTAssertEqual(run.runs, [DatabaseAIRowRun(rowId: "93e5f9e5-5f6e-4c40-ab8c-9ac28e28c524", runId: "d93a25ed-1f05-4353-ab79-f8f1e60082be", queued: false, skipped: DatabaseAISkipReason.userEdited)])318        XCTAssertEqual(run.skipped?.first?.reason, DatabaseAISkipReason.userEdited)319320        let log = try NextFixture.decode([DatabaseAIRun].self, "ai-doc", "database-ai-runs")321        let entry = try XCTUnwrap(log.first)322        XCTAssertEqual(entry.status, "done")323        XCTAssertEqual(entry.trigger, "auto")324        XCTAssertEqual(entry.task, DatabaseAITask.classify)325        XCTAssertEqual(entry.inputTokens, 561)326        XCTAssertEqual(entry.outputTokens, 13)327        XCTAssertEqual(entry.costUsd, 0.0000626)328        XCTAssertNil(entry.error)329        XCTAssertLessThan(try XCTUnwrap(entry.startedAt), try XCTUnwrap(entry.finishedAt))330    }331332    func testEveryDocumentFixtureIsUsed() throws {333        // Guards against a transcribed example nobody decodes.334        let used: Set<String> = ["models-providers", "models-creators-first-two", "model-openrouter", "chat-page-meta", "chat-page-done", "chat-none-meta",335                                 "chat-none-done", "chat-error-page-required", "action-meta", "action-links-result", "action-links-done", "zoom-ai-ready",336                                 "zoom-ai-missing", "audio-transcribe-queued", "audio-done", "database-ai-config", "database-ai-config-put",337                                 "database-ai-run-skipped", "database-ai-runs"]338        XCTAssertEqual(Set(try NextFixture.index("ai-doc").map(\.name)), used)339    }340}341
Files main
Menu

Theme