SPB Git

Repository Public

KnowledgeTestSupport.swift 6.7 KB 127 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeTestSupport.swift3// Description: Generators used by the Knowledge tests: real PDF files, minimal valid DOCX (ZIP) files, a deterministic fake embedder.4import Compression5import CoreGraphics6import CoreText7import Foundation8import LocalChatCore9import Synchronization10import zlib11@testable import LocalChatKnowledge1213enum KnowledgeFixtures {14    /// A PDF with a real text layer, one page per string.15    static func pdf(pages: [String]) -> Data {16        let data = NSMutableData()17        var box = CGRect(x: 0, y: 0, width: 612, height: 792)18        let context = CGContext(consumer: CGDataConsumer(data: data as CFMutableData)!, mediaBox: &box, nil)!19        for text in pages {20            context.beginPDFPage(nil)21            if !text.isEmpty { draw(text, in: context, size: 14, origin: CGPoint(x: 72, y: 700)) }22            context.endPDFPage()23        }24        context.closePDF()25        return data as Data26    }2728    /// A PDF whose single page is a bitmap of the text (no text layer): what a scanner produces.29    static func scannedPDF(text: String) -> Data {30        let width = 1224, height = 40031        let bitmap = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),32                               bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!33        bitmap.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1)); bitmap.fill(CGRect(x: 0, y: 0, width: width, height: height))34        draw(text, in: bitmap, size: 56, origin: CGPoint(x: 60, y: 180))35        let image = bitmap.makeImage()!36        let data = NSMutableData()37        var box = CGRect(x: 0, y: 0, width: 612, height: 200)38        let context = CGContext(consumer: CGDataConsumer(data: data as CFMutableData)!, mediaBox: &box, nil)!39        context.beginPDFPage(nil); context.draw(image, in: box); context.endPDFPage(); context.closePDF()40        return data as Data41    }4243    private static func draw(_ text: String, in context: CGContext, size: CGFloat, origin: CGPoint) {44        let font = CTFontCreateWithName("Helvetica" as CFString, size, nil)45        var y = origin.y46        for line in text.split(separator: "\n", omittingEmptySubsequences: false) {47            let attributed = NSAttributedString(string: String(line), attributes: [NSAttributedString.Key(kCTFontAttributeName as String): font,48                                                                                    NSAttributedString.Key(kCTForegroundColorAttributeName as String): CGColor(red: 0, green: 0, blue: 0, alpha: 1)])49            context.textPosition = CGPoint(x: origin.x, y: y)50            CTLineDraw(CTLineCreateWithAttributedString(attributed), context)51            y -= size * 1.452        }53    }5455    static func documentXML(_ body: String) -> String {56        "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:body>\(body)</w:body></w:document>"57    }58    static func paragraph(_ text: String, style: String? = nil) -> String {59        "<w:p>\(style.map { "<w:pPr><w:pStyle w:val=\"\($0)\"/></w:pPr>" } ?? "")<w:r><w:t xml:space=\"preserve\">\(text)</w:t></w:r></w:p>"60    }6162    /// A valid ZIP archive (local headers, central directory, end record) with stored or raw-DEFLATE entries.63    static func zip(_ files: [(name: String, data: Data)], deflate: Bool, flags: UInt16 = 0, corruptCRC: Bool = false) -> Data {64        var out = Data(); var central = Data()65        func le16(_ v: Int) -> Data { withUnsafeBytes(of: UInt16(v).littleEndian) { Data($0) } }66        func le32(_ v: Int) -> Data { withUnsafeBytes(of: UInt32(truncatingIfNeeded: v).littleEndian) { Data($0) } }67        for file in files {68            var crc = file.data.withUnsafeBytes { Int(crc32(0, $0.bindMemory(to: UInt8.self).baseAddress, UInt32($0.count))) }69            if corruptCRC { crc ^= 0xFF }70            var payload = file.data; var method = 071            if deflate {72                var buffer = Data(count: file.data.count + 1024)73                let n = buffer.withUnsafeMutableBytes { dst in74                    file.data.withUnsafeBytes { src in75                        compression_encode_buffer(dst.bindMemory(to: UInt8.self).baseAddress!, dst.count, src.bindMemory(to: UInt8.self).baseAddress!, src.count, nil, COMPRESSION_ZLIB)76                    }77                }78                precondition(n > 0); payload = buffer.prefix(n); method = 879            }80            let name = Data(file.name.utf8); let offset = out.count81            let common = le16(Int(flags)) + le16(method) + le16(0) + le16(0) + le32(crc) + le32(payload.count) + le32(file.data.count) + le16(name.count) + le16(0)82            out += le32(0x0403_4b50) + le16(20) + common + name + payload83            central += le32(0x0201_4b50) + le16(20) + le16(20) + common + le16(0) + le16(0) + le16(0) + le32(0) + le32(offset) + name84        }85        let cdOffset = out.count86        out += central + le32(0x0605_4b50) + le16(0) + le16(0) + le16(files.count) + le16(files.count) + le32(central.count) + le32(cdOffset) + le16(0)87        return out88    }8990    static func docx(body: String, deflate: Bool) -> Data {91        zip([("[Content_Types].xml", Data("<Types/>".utf8)), ("word/document.xml", Data(documentXML(body).utf8))], deflate: deflate)92    }93}9495/// Deterministic bag-of-words embedder: texts sharing words have a high cosine similarity.96final class FakeEmbedder: EmbeddingProvider {97    let identifier: String98    let dimension: Int99    let isRemote: Bool100    let delay: Duration101    let failing: Bool102    private let callCount = Mutex(0)103    var calls: Int { callCount.withLock { $0 } }104105    init(identifier: String = "fake.bow", dimension: Int = 64, isRemote: Bool = false, delay: Duration = .zero, failing: Bool = false) {106        self.identifier = identifier; self.dimension = dimension; self.isRemote = isRemote; self.delay = delay; self.failing = failing107    }108    struct Failure: Error {}109110    static func vector(_ text: String, dimension: Int) -> [Float] {111        var v = [Float](repeating: 0, count: dimension)112        for word in text.lowercased().split(whereSeparator: { !($0.isLetter || $0.isNumber) }) where word.count > 2 {113            var h: UInt64 = 1469598103934665603114            for b in word.utf8 { h = (h ^ UInt64(b)) &* 1099511628211 }115            v[Int(h % UInt64(dimension))] += 1116        }117        return v118    }119120    func embed(_ texts: [String]) async throws -> [[Float]] {121        callCount.withLock { $0 += 1 }122        if delay > .zero { try await Task.sleep(for: delay) }123        if failing { throw Failure() }124        return texts.map { Self.vector($0, dimension: dimension) }125    }126}127
Files main
Menu

Theme