SPB Git

Repository Public

SSEAndAITests.swift 15.1 KB 280 lines swift

1// SP Notes Core2// File: Tests/SPNotesAPITests/SSEAndAITests.swift3// Description: The SSE parser fuzzed on random chunk boundaries (inside UTF-8 sequences and CRLF pairs), AI event4//              decoding, and the AI stream end to end on the stub (pre-stream errors, events, cancellation).5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import XCTest10@testable import SPNotesAPI1112final class SSEAndAITests: XCTestCase {13    private func parse(_ text: String, finish: Bool = false) -> [SSEEvent] {14        var parser = SSEParser()15        var events = parser.feed(Array(text.utf8))16        if finish, let last = parser.finish() { events.append(last) }17        return events18    }1920    func testBasicEventsAndDone() {21        let events = parse("data: {\"type\":\"text\",\"delta\":\"a\"}\n\ndata: [DONE]\n\n")22        XCTAssertEqual(events, [SSEEvent(data: "{\"type\":\"text\",\"delta\":\"a\"}"), SSEEvent(data: "[DONE]")])23        XCTAssertFalse(events[0].isDone)24        XCTAssertTrue(events[1].isDone)25    }2627    func testLineEndingsFieldsCommentsAndMultilineData() {28        XCTAssertEqual(parse("data: a\r\n\r\ndata: b\r\rdata: c\n\n"), [SSEEvent(data: "a"), SSEEvent(data: "b"), SSEEvent(data: "c")])29        XCTAssertEqual(parse("event: update\nid: 7\ndata: one\ndata: two\n\n"), [SSEEvent(event: "update", data: "one\ntwo", id: "7")])30        XCTAssertEqual(parse(": ping\n\n:\n\ndata: x\n\n"), [SSEEvent(data: "x")])31        XCTAssertEqual(parse("data:no-space\n\ndata:  two-spaces\n\ndata\n\n"), [SSEEvent(data: "no-space"), SSEEvent(data: " two-spaces"), SSEEvent(data: "")])32        XCTAssertEqual(parse("retry: 10\nunknown: y\n\nevent: only\n\ndata: z\n\n"), [SSEEvent(data: "z")], "blocks without data dispatch nothing and do not leak their event name")33        XCTAssertEqual(parse("\u{FEFF}data: bom\n\n"), [SSEEvent(data: "bom")])34        XCTAssertEqual(parse("id: 1\ndata: a\n\ndata: b\n\n"), [SSEEvent(data: "a", id: "1"), SSEEvent(data: "b", id: "1")], "the last event id persists")35        XCTAssertEqual(parse("data: a: b: c\n\n"), [SSEEvent(data: "a: b: c")])36        XCTAssertEqual(parse("\n\n\n"), [])37    }3839    func testUnterminatedTailIsFlushedOnFinish() {40        XCTAssertEqual(parse("data: a\n\ndata: [DONE]"), [SSEEvent(data: "a")])41        XCTAssertEqual(parse("data: a\n\ndata: [DONE]", finish: true), [SSEEvent(data: "a"), SSEEvent(data: "[DONE]")])42        XCTAssertEqual(parse("data: a\n\ndata: tail\n", finish: true), [SSEEvent(data: "a"), SSEEvent(data: "tail")])43        XCTAssertEqual(parse("data: a\n\n", finish: true), [SSEEvent(data: "a")])44    }4546    func testFuzzRandomChunkSplits() {47        var generator = SystemRandomNumberGenerator()48        let samples = ["é", "日本語のテキスト", "😀👨‍👩‍👧‍👦🇨🇦", "plain ascii", "tab\there", "{\"type\":\"text\",\"delta\":\"ça va — très bien ✓\"}", "", "a: b", " leading", "x\u{0301}y"]49        let lineEnds = ["\n", "\r\n", "\r"]50        for round in 0..<300 {51            var expected: [SSEEvent] = []52            var wire = ""53            var lastID: String?54            let count = Int.random(in: 1...12, using: &generator)55            for index in 0..<count {56                let eol = lineEnds.randomElement(using: &generator)!57                if Bool.random(using: &generator) { wire += ": heartbeat \(index)\(eol)" }58                var name = "message"59                if Int.random(in: 0..<4, using: &generator) == 0 { name = "kind\(index)"; wire += "event: \(name)\(eol)" }60                if Int.random(in: 0..<5, using: &generator) == 0 { lastID = "id-\(round)-\(index)"; wire += "id: \(lastID!)\(eol)" }61                let lines = (0..<Int.random(in: 1...3, using: &generator)).map { _ in samples.randomElement(using: &generator)! }62                for line in lines { wire += "data: \(line)\(eol)" }63                wire += eol64                expected.append(SSEEvent(event: name, data: lines.joined(separator: "\n"), id: lastID))65            }66            wire += "data: [DONE]\n\n"67            expected.append(SSEEvent(data: "[DONE]", id: lastID))68            let bytes = Array(wire.utf8)6970            // Whole buffer, byte by byte, and random chunkings (cuts land inside UTF-8 sequences and CRLF pairs).71            XCTAssertEqual(parse(wire), expected, "whole, round \(round)")72            var single = SSEParser()73            var bytewise: [SSEEvent] = []74            for byte in bytes { if let event = single.feed(byte: byte) { bytewise.append(event) } }75            XCTAssertEqual(bytewise, expected, "bytewise, round \(round)")76            for _ in 0..<4 {77                var parser = SSEParser()78                var got: [SSEEvent] = []79                var offset = 080                while offset < bytes.count {81                    let size = min(bytes.count - offset, Int.random(in: 1...9, using: &generator))82                    got += parser.feed(bytes[offset..<(offset + size)])83                    offset += size84                }85                XCTAssertNil(parser.finish())86                XCTAssertEqual(got, expected, "chunked, round \(round): \(wire.debugDescription)")87            }88        }89    }9091    func testSplitInsideEveryByteOfAMultibyteStream() {92        let wire = "data: {\"type\":\"text\",\"delta\":\"déjà 😀 vu\"}\r\n\r\ndata: [DONE]\r\n\r\n"93        let bytes = Array(wire.utf8)94        for cut in 0...bytes.count {95            var parser = SSEParser()96            let got = parser.feed(bytes[..<cut]) + parser.feed(bytes[cut...])97            XCTAssertEqual(got.map(\.data), ["{\"type\":\"text\",\"delta\":\"déjà 😀 vu\"}", "[DONE]"], "cut at \(cut)")98        }99    }100101    // MARK: AI events102103    func testAIEventDecoding() throws {104        func event(_ json: String) -> AIEvent? { AIEvent(json: Data(json.utf8)) }105        guard case .meta(let meta)? = event(#"{"type":"meta","model":"claude-haiku","provider":"anthropic","reason":"default","budgetNote":null,"conversationId":"c1","sources":[]}"#) else { return XCTFail("meta") }106        XCTAssertEqual(meta.model, "claude-haiku")107        XCTAssertEqual(meta.provider, "anthropic")108        XCTAssertEqual(meta.reason, "default")109        XCTAssertNil(meta.budgetNote)110        XCTAssertEqual(meta.conversationId, "c1")111        XCTAssertTrue(meta.raw.string.contains("\"sources\":[]"))112113        XCTAssertEqual(event(#"{"type":"text","delta":"Bonjour é"}"#), .text("Bonjour é"))114115        guard case .done(let done)? = event(#"{"type":"done","usage":{"input":12,"output":34},"costUsd":0.0012,"latencyMs":850,"messageId":"m1","content":"final"}"#) else { return XCTFail("done") }116        XCTAssertEqual(done.costUsd, 0.0012)117        XCTAssertEqual(done.latencyMs, 850)118        XCTAssertEqual(done.usage, RawJSON(string: #"{"input":12,"output":34}"#))119        XCTAssertEqual(done.messageId, "m1")120        XCTAssertEqual(done.content, "final")121        guard case .done(let bare)? = event(#"{"type":"done","usage":null,"costUsd":null,"latencyMs":0}"#) else { return XCTFail("done") }122        XCTAssertNil(bare.usage)123        XCTAssertNil(bare.costUsd)124125        guard case .error(let failure)? = event(#"{"type":"error","code":"provider_error","retryable":true,"message":"Overloaded"}"#) else { return XCTFail("error") }126        XCTAssertEqual(failure.code, "provider_error")127        XCTAssertTrue(failure.retryable)128        XCTAssertEqual(failure.message, "Overloaded")129130        guard case .needsConfirmation(let confirmation)? = event(#"{"type":"needs_confirmation","reason":"Long context","plan":{"estCostUsd":1.2}}"#) else { return XCTFail("needs_confirmation") }131        XCTAssertEqual(confirmation.reason, "Long context")132        XCTAssertEqual(confirmation.plan, RawJSON(string: #"{"estCostUsd":1.2}"#))133134        guard case .other(let type, let raw)? = event(#"{"type":"sources","items":[1]}"#) else { return XCTFail("other") }135        XCTAssertEqual(type, "sources")136        XCTAssertEqual(raw.string, #"{"type":"sources","items":[1]}"#)137138        XCTAssertNil(event("[DONE]"))139        XCTAssertNil(event(#"{"delta":"no type"}"#))140        XCTAssertNil(event("not json"))141    }142143    private static let streamBody = [144        #"data: {"type":"meta","model":"m","provider":"p","reason":"r","budgetNote":null}"#, "",145        #"data: {"type":"text","delta":"Bon"}"#, "",146        ": keep-alive", "",147        #"data: {"type":"text","delta":"jour 😀"}"#, "",148        #"data: {"type":"done","usage":{"output":2},"costUsd":0.001,"latencyMs":12}"#, "",149        "data: [DONE]", "",150        #"data: {"type":"text","delta":"after the terminator: never delivered"}"#, "", "",151    ].joined(separator: "\n")152153    func testAIActionStreamEndToEnd() async throws {154        StubServer.shared.reset { _ in StubReply(status: 200, headers: ["Content-Type": "text/event-stream; charset=utf-8"], body: Data(Self.streamBody.utf8)) }155        let (api, _, _) = TestEnv.api()156        var text = ""157        var kinds: [String] = []158        for try await event in try await api.aiAction(AIActionRequest(action: .summarize, text: "some text", language: nil)) {159            switch event {160            case .meta(let meta): kinds.append("meta"); XCTAssertEqual(meta.model, "m")161            case .text(let delta): kinds.append("text"); text += delta162            case .done(let done): kinds.append("done"); XCTAssertEqual(done.costUsd, 0.001)163            default: kinds.append("other")164            }165        }166        XCTAssertEqual(kinds, ["meta", "text", "text", "done"])167        XCTAssertEqual(text, "Bonjour 😀")168        let request = try XCTUnwrap(StubServer.shared.last)169        XCTAssertEqual(request.method, "POST")170        XCTAssertEqual(request.path, "/api/ai/action")171        XCTAssertEqual(request.header("Accept"), "text/event-stream")172        XCTAssertEqual(request.header("Origin"), "https://notes.test")173        XCTAssertEqual(request.bodyText, #"{"action":"summarize","text":"some text"}"#)174    }175176    func testAIChatStreamCarriesInStreamErrorsAndConfirmation() async throws {177        let body = "data: {\"type\":\"needs_confirmation\",\"reason\":\"big\",\"plan\":{\"estTokens\":90000}}\n\ndata: {\"type\":\"error\",\"code\":\"refused\",\"retryable\":true,\"message\":\"declined\"}\n\ndata: [DONE]\n\n"178        StubServer.shared.reset { _ in StubReply(status: 200, headers: ["Content-Type": "text/event-stream"], body: Data(body.utf8)) }179        let (api, _, _) = TestEnv.api()180        var events: [AIEvent] = []181        for try await event in try await api.aiChat(AIChatRequest(message: "hello", clientId: "client-0001", mode: .general)) { events.append(event) }182        XCTAssertEqual(events.count, 2)183        guard case .needsConfirmation(let confirmation) = events[0], case .error(let failure) = events[1] else { return XCTFail("\(events)") }184        XCTAssertEqual(confirmation.plan, RawJSON(string: #"{"estTokens":90000}"#))185        XCTAssertEqual(failure.code, "refused")186        XCTAssertEqual(StubServer.shared.last?.path, "/api/ai/chat")187        XCTAssertEqual(StubServer.shared.last?.bodyText, #"{"clientId":"client-0001","message":"hello","mode":"general"}"#)188    }189190    func testPreStreamErrorsAreThrownTyped() async {191        let (api, _, _) = TestEnv.api()192193        StubServer.shared.reset { _ in .json(#"{"error":"About $0.90 for this request","code":"confirm_expensive","plan":{"estCostUsd":0.9,"task":"deep"}}"#, status: 409) }194        var error = await captureAPIError { try await api.aiAction(AIActionRequest(action: .rewrite, text: "x")) }195        guard case .confirmExpensive(let plan, let message) = error else { return XCTFail("\(String(describing: error))") }196        XCTAssertEqual(message, "About $0.90 for this request")197        XCTAssertEqual(plan, RawJSON(string: #"{"task":"deep","estCostUsd":0.9}"#))198199        StubServer.shared.reset { _ in .json(#"{"error":"AI is not configured","code":"not_configured"}"#, status: 503) }200        error = await captureAPIError { try await api.aiAction(AIActionRequest(action: .rewrite, text: "x")) }201        XCTAssertEqual(error?.code, "not_configured")202        XCTAssertEqual(error?.status, 503)203        XCTAssertEqual(error?.isRetryable, false)204205        StubServer.shared.reset { _ in .json(#"{"error":"No model","code":"no_model"}"#, status: 503) }206        error = await captureAPIError { try await api.aiChat(AIChatRequest(message: "x")) }207        XCTAssertEqual(error?.code, "no_model")208        XCTAssertEqual(error?.isRetryable, false)209210        StubServer.shared.reset { _ in .json(#"{"error":"Write an instruction first","code":"no_instruction"}"#, status: 400) }211        error = await captureAPIError { try await api.aiAction(AIActionRequest(action: .custom, text: "x")) }212        XCTAssertEqual(error?.status, 400)213        XCTAssertEqual(error?.message, "Write an instruction first")214215        StubServer.shared.reset { _ in .failure(.notConnectedToInternet) }216        error = await captureAPIError { try await api.aiAction(AIActionRequest(action: .rewrite, text: "x")) }217        XCTAssertEqual(error, .offline)218    }219220    /// An endless byte source that records when its consumer goes away.221    private struct EndlessEvents: AsyncSequence, Sendable {222        typealias Element = UInt8223        let cancelled: Flag224225        final class Flag: @unchecked Sendable {226            private let lock = NSLock()227            private var value = false228            func set() { lock.withLock { value = true } }229            var isSet: Bool { lock.withLock { value } }230        }231232        struct Iterator: AsyncIteratorProtocol {233            let cancelled: Flag234            var buffer: [UInt8] = []235            var index = 0236            var count = 0237238            mutating func next() async throws -> UInt8? {239                if index == buffer.count {240                    do { try await Task.sleep(nanoseconds: 2_000_000) } catch { cancelled.set(); throw error }241                    count += 1242                    buffer = Array("data: {\"type\":\"text\",\"delta\":\"\(count) \"}\n\n".utf8)243                    index = 0244                }245                defer { index += 1 }246                return buffer[index]247            }248        }249250        func makeAsyncIterator() -> Iterator { Iterator(cancelled: cancelled) }251    }252253    func testCancellingTheConsumerCancelsTheReader() async throws {254        let flag = EndlessEvents.Flag()255        let stream = NotesAPI.events(from: EndlessEvents(cancelled: flag))256        let consumer = Task { () -> Int in257            var received = 0258            do { for try await _ in stream { received += 1 } } catch {}259            return received260        }261        try await Task.sleep(nanoseconds: 60_000_000)262        XCTAssertFalse(flag.isSet)263        consumer.cancel()264        let received = await consumer.value265        XCTAssertGreaterThan(received, 0)266        for _ in 0..<200 where !flag.isSet { try await Task.sleep(nanoseconds: 5_000_000) }267        XCTAssertTrue(flag.isSet, "the task reading the response must be cancelled with its consumer")268    }269270    func testBreakingOutOfTheLoopCancelsTheReader() async throws {271        let flag = EndlessEvents.Flag()272        do {273            let stream = NotesAPI.events(from: EndlessEvents(cancelled: flag))274            for try await _ in stream { break }275        }276        for _ in 0..<200 where !flag.isSet { try await Task.sleep(nanoseconds: 5_000_000) }277        XCTAssertTrue(flag.isSet, "dropping the stream must close the connection")278    }279}280
Files main
Menu

Theme