Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeTestSupport.swift3// Description: Generators used by the Knowledge tests: real PDF files, minimal valid DOCX (ZIP) files, a deterministic fake embedder.4import Compression5import CoreGraphics6import CoreText7import Foundation8import LocalChatCore9import Synchronization10import zlib11@testable import LocalChatKnowledge1213enum KnowledgeFixtures {14 /// A PDF with a real text layer, one page per string.15 static func pdf(pages: [String]) -> Data {16 let data = NSMutableData()17 var box = CGRect(x: 0, y: 0, width: 612, height: 792)18 let context = CGContext(consumer: CGDataConsumer(data: data as CFMutableData)!, mediaBox: &box, nil)!19 for text in pages {20 context.beginPDFPage(nil)21 if !text.isEmpty { draw(text, in: context, size: 14, origin: CGPoint(x: 72, y: 700)) }22 context.endPDFPage()23 }24 context.closePDF()25 return data as Data26 }2728 /// A PDF whose single page is a bitmap of the text (no text layer): what a scanner produces.29 static func scannedPDF(text: String) -> Data {30 let width = 1224, height = 40031 let bitmap = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),32 bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!33 bitmap.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1)); bitmap.fill(CGRect(x: 0, y: 0, width: width, height: height))34 draw(text, in: bitmap, size: 56, origin: CGPoint(x: 60, y: 180))35 let image = bitmap.makeImage()!36 let data = NSMutableData()37 var box = CGRect(x: 0, y: 0, width: 612, height: 200)38 let context = CGContext(consumer: CGDataConsumer(data: data as CFMutableData)!, mediaBox: &box, nil)!39 context.beginPDFPage(nil); context.draw(image, in: box); context.endPDFPage(); context.closePDF()40 return data as Data41 }4243 private static func draw(_ text: String, in context: CGContext, size: CGFloat, origin: CGPoint) {44 let font = CTFontCreateWithName("Helvetica" as CFString, size, nil)45 var y = origin.y46 for line in text.split(separator: "\n", omittingEmptySubsequences: false) {47 let attributed = NSAttributedString(string: String(line), attributes: [NSAttributedString.Key(kCTFontAttributeName as String): font,48 NSAttributedString.Key(kCTForegroundColorAttributeName as String): CGColor(red: 0, green: 0, blue: 0, alpha: 1)])49 context.textPosition = CGPoint(x: origin.x, y: y)50 CTLineDraw(CTLineCreateWithAttributedString(attributed), context)51 y -= size * 1.452 }53 }5455 static func documentXML(_ body: String) -> String {56 "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:body>\(body)</w:body></w:document>"57 }58 static func paragraph(_ text: String, style: String? = nil) -> String {59 "<w:p>\(style.map { "<w:pPr><w:pStyle w:val=\"\($0)\"/></w:pPr>" } ?? "")<w:r><w:t xml:space=\"preserve\">\(text)</w:t></w:r></w:p>"60 }6162 /// A valid ZIP archive (local headers, central directory, end record) with stored or raw-DEFLATE entries.63 static func zip(_ files: [(name: String, data: Data)], deflate: Bool, flags: UInt16 = 0, corruptCRC: Bool = false) -> Data {64 var out = Data(); var central = Data()65 func le16(_ v: Int) -> Data { withUnsafeBytes(of: UInt16(v).littleEndian) { Data($0) } }66 func le32(_ v: Int) -> Data { withUnsafeBytes(of: UInt32(truncatingIfNeeded: v).littleEndian) { Data($0) } }67 for file in files {68 var crc = file.data.withUnsafeBytes { Int(crc32(0, $0.bindMemory(to: UInt8.self).baseAddress, UInt32($0.count))) }69 if corruptCRC { crc ^= 0xFF }70 var payload = file.data; var method = 071 if deflate {72 var buffer = Data(count: file.data.count + 1024)73 let n = buffer.withUnsafeMutableBytes { dst in74 file.data.withUnsafeBytes { src in75 compression_encode_buffer(dst.bindMemory(to: UInt8.self).baseAddress!, dst.count, src.bindMemory(to: UInt8.self).baseAddress!, src.count, nil, COMPRESSION_ZLIB)76 }77 }78 precondition(n > 0); payload = buffer.prefix(n); method = 879 }80 let name = Data(file.name.utf8); let offset = out.count81 let common = le16(Int(flags)) + le16(method) + le16(0) + le16(0) + le32(crc) + le32(payload.count) + le32(file.data.count) + le16(name.count) + le16(0)82 out += le32(0x0403_4b50) + le16(20) + common + name + payload83 central += le32(0x0201_4b50) + le16(20) + le16(20) + common + le16(0) + le16(0) + le16(0) + le32(0) + le32(offset) + name84 }85 let cdOffset = out.count86 out += central + le32(0x0605_4b50) + le16(0) + le16(0) + le16(files.count) + le16(files.count) + le32(central.count) + le32(cdOffset) + le16(0)87 return out88 }8990 static func docx(body: String, deflate: Bool) -> Data {91 zip([("[Content_Types].xml", Data("<Types/>".utf8)), ("word/document.xml", Data(documentXML(body).utf8))], deflate: deflate)92 }93}9495/// Deterministic bag-of-words embedder: texts sharing words have a high cosine similarity.96final class FakeEmbedder: EmbeddingProvider {97 let identifier: String98 let dimension: Int99 let isRemote: Bool100 let delay: Duration101 let failing: Bool102 private let callCount = Mutex(0)103 var calls: Int { callCount.withLock { $0 } }104105 init(identifier: String = "fake.bow", dimension: Int = 64, isRemote: Bool = false, delay: Duration = .zero, failing: Bool = false) {106 self.identifier = identifier; self.dimension = dimension; self.isRemote = isRemote; self.delay = delay; self.failing = failing107 }108 struct Failure: Error {}109110 static func vector(_ text: String, dimension: Int) -> [Float] {111 var v = [Float](repeating: 0, count: dimension)112 for word in text.lowercased().split(whereSeparator: { !($0.isLetter || $0.isNumber) }) where word.count > 2 {113 var h: UInt64 = 1469598103934665603114 for b in word.utf8 { h = (h ^ UInt64(b)) &* 1099511628211 }115 v[Int(h % UInt64(dimension))] += 1116 }117 return v118 }119120 func embed(_ texts: [String]) async throws -> [[Float]] {121 callCount.withLock { $0 += 1 }122 if delay > .zero { try await Task.sleep(for: delay) }123 if failing { throw Failure() }124 return texts.map { Self.vector($0, dimension: dimension) }125 }126}127