Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatKnowledge/Sources/KnowledgeIndexer.swift3// Description: Document pipeline: hash → extract → chunk → embed in batches → one transactional write. Deduplicates, reports progress, cancels cleanly, re-indexes.4import CryptoKit5import Foundation6import LocalChatCore78public struct IndexingProgress: Sendable, Hashable {9 public enum Stage: String, Sendable { case extracting, chunking, embedding, writing, finished, duplicate, failed, cancelled }10 public var documentID: UUID11 public var documentName: String12 public var collectionID: UUID13 public var stage: Stage14 public var chunksDone: Int15 public var chunksTotal: Int16 public var message: String?17 public var fraction: Double? { chunksTotal > 0 ? Double(chunksDone) / Double(chunksTotal) : nil }18}1920public enum KnowledgeIndexingError: LocalizedError, Sendable, Equatable {21 case remoteProcessingNotAllowed(String)22 case embedderMismatch(collection: String, provider: String)23 case embeddingFailed(String)24 case nothingToIndex25 public var errorDescription: String? {26 switch self {27 case .remoteProcessingNotAllowed(let p): "“\(p)” runs on another machine. Your documents stay on this device unless you explicitly allow remote processing."28 case .embedderMismatch(let c, let p): "This collection was indexed with “\(c)”. Re-index the collection to switch to “\(p)”; vectors from two models cannot be mixed."29 case .embeddingFailed(let m): "The embedding model failed: \(m)"30 case .nothingToIndex: "The document produced no text to index."31 }32 }33}3435public actor KnowledgeIndexer {36 public let store: KnowledgeStore37 public let extractor: TextExtractor38 public let batchSize: Int39 private var observers: [UUID: AsyncStream<IndexingProgress>.Continuation] = [:]40 private var busyHashes: Set<String> = []4142 public init(store: KnowledgeStore, extractor: TextExtractor = .init(), batchSize: Int = 32) {43 self.store = store; self.extractor = extractor; self.batchSize = max(1, batchSize)44 }4546 /// Every progress event of this indexer, for as long as the returned stream is iterated.47 public func progressUpdates() -> AsyncStream<IndexingProgress> {48 let id = UUID()49 let (stream, continuation) = AsyncStream<IndexingProgress>.makeStream(bufferingPolicy: .bufferingNewest(512))50 observers[id] = continuation51 continuation.onTermination = { [weak self] _ in Task { await self?.removeObserver(id) } }52 return stream53 }54 private func removeObserver(_ id: UUID) { observers[id] = nil }55 private func report(_ d: KnowledgeDocument, _ stage: IndexingProgress.Stage, _ done: Int = 0, _ total: Int = 0, _ message: String? = nil) {56 let event = IndexingProgress(documentID: d.id, documentName: d.name, collectionID: d.collectionID, stage: stage, chunksDone: done, chunksTotal: total, message: message)57 for c in observers.values { c.yield(event) }58 }5960 @discardableResult61 public func add(fileURL: URL, to collectionID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws -> KnowledgeDocument {62 let scoped = fileURL.startAccessingSecurityScopedResource()63 defer { if scoped { fileURL.stopAccessingSecurityScopedResource() } }64 if let size = try? fileURL.resourceValues(forKeys: [.fileSizeKey]).fileSize, size > extractor.options.maxBytes {65 throw ExtractionError.tooLarge(bytes: size, limit: extractor.options.maxBytes)66 }67 let data: Data68 do { data = try Data(contentsOf: fileURL, options: .mappedIfSafe) } catch { throw ExtractionError.corrupt("cannot read the file (\(error.localizedDescription))") }69 return try await add(data: data, name: fileURL.lastPathComponent, to: collectionID, provider: provider, allowRemoteProcessing: allowRemoteProcessing)70 }7172 /// Adds a document. An identical file (same SHA-256) already indexed in the collection is returned as is.73 @discardableResult74 public func add(data: Data, name: String, to collectionID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws -> KnowledgeDocument {75 try Self.checkPrivacy(provider, allowRemoteProcessing)76 guard var collection = try store.collection(collectionID) else { throw KnowledgeStoreError.collectionNotFound }77 let hash = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined()78 let busyKey = collectionID.uuidString + hash79 var document: KnowledgeDocument80 if let existing = try store.document(inCollection: collectionID, contentHash: hash) {81 if existing.status != .failed || busyHashes.contains(busyKey) { report(existing, .duplicate, existing.chunkCount, existing.chunkCount, "Already in this collection."); return existing }82 document = existing; document.name = name // retry a failed import in place83 } else {84 document = KnowledgeDocument(collectionID: collectionID, name: name, kind: TextExtractor.kind(forFileName: name)?.kind.rawValue ?? "txt", byteCount: data.count, contentHash: hash)85 }86 // vectors of two embedding models are never mixed: an empty collection adopts the provider, otherwise it must match87 if collection.embedderID != provider.identifier || collection.dimension != provider.dimension {88 guard try store.counts(inCollection: collectionID).vectors == 0 else { throw KnowledgeIndexingError.embedderMismatch(collection: collection.embedderID, provider: provider.identifier) }89 collection.embedderID = provider.identifier; collection.dimension = provider.dimension; collection.updatedAt = Date()90 try store.saveCollection(collection)91 }92 document.status = .indexing; document.errorText = nil93 try store.saveDocument(document) // visible to concurrent adds of the same file94 busyHashes.insert(busyKey); defer { busyHashes.remove(busyKey) }95 do {96 report(document, .extracting)97 let extractor = extractor98 let extracted = try await Task.detached(priority: .userInitiated) { try extractor.extract(data: data, fileName: name) }.value99 document.kind = extracted.kind.rawValue; document.characterCount = extracted.characterCount100 try Task.checkCancellation()101 report(document, .chunking)102 let chunks = Chunker(collection: collection).chunk(extracted)103 guard !chunks.isEmpty else { throw KnowledgeIndexingError.nothingToIndex }104 let vectors = try await embed(chunks, provider, document)105 report(document, .writing, chunks.count, chunks.count)106 try store.replaceIndex([.init(document: document, extracted: extracted, chunks: chunks, vectors: vectors)])107 let saved = try store.document(document.id) ?? document108 report(saved, .finished, chunks.count, chunks.count, extracted.warnings.first)109 return saved110 } catch is CancellationError {111 try? store.deleteDocument(document.id) // nothing partial is left behind112 report(document, .cancelled)113 throw CancellationError()114 } catch {115 document.status = .failed; document.chunkCount = 0116 document.errorText = (error as? LocalizedError)?.errorDescription ?? String(describing: error)117 try? store.saveDocument(document)118 report(document, .failed, 0, 0, document.errorText)119 throw error120 }121 }122123 /// Re-chunks (with the collection's current settings) and re-embeds one document from its stored text. The previous index stays in place until the new one is written.124 @discardableResult125 public func reindex(documentID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws -> KnowledgeDocument {126 try Self.checkPrivacy(provider, allowRemoteProcessing)127 guard let document = try store.document(documentID) else { throw KnowledgeStoreError.documentNotFound }128 guard let collection = try store.collection(document.collectionID) else { throw KnowledgeStoreError.collectionNotFound }129 guard collection.embedderID == provider.identifier, collection.dimension == provider.dimension else {130 throw KnowledgeIndexingError.embedderMismatch(collection: collection.embedderID, provider: provider.identifier)131 }132 guard let extracted = try store.extractedDocument(documentID), !extracted.sections.isEmpty else { throw KnowledgeIndexingError.nothingToIndex }133 do {134 let chunks = Chunker(collection: collection).chunk(extracted)135 guard !chunks.isEmpty else { throw KnowledgeIndexingError.nothingToIndex }136 let vectors = try await embed(chunks, provider, document)137 report(document, .writing, chunks.count, chunks.count)138 try store.replaceIndex([.init(document: document, extracted: nil, chunks: chunks, vectors: vectors)])139 let saved = try store.document(documentID) ?? document140 report(saved, .finished, chunks.count, chunks.count)141 return saved142 } catch is CancellationError { report(document, .cancelled); throw CancellationError() }143 }144145 /// Re-indexes every document of the collection with `provider` and adopts it (identifier, dimension) in one transaction:146 /// the collection never holds vectors of two models, and a cancellation or failure leaves the previous index untouched.147 public func reindex(collectionID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws {148 try Self.checkPrivacy(provider, allowRemoteProcessing)149 guard var collection = try store.collection(collectionID) else { throw KnowledgeStoreError.collectionNotFound }150 collection.embedderID = provider.identifier; collection.dimension = provider.dimension; collection.updatedAt = Date()151 var contents: [KnowledgeStore.IndexedContent] = []152 let chunker = Chunker(collection: collection)153 for document in try store.documents(in: collectionID) {154 guard let extracted = try store.extractedDocument(document.id), !extracted.sections.isEmpty else { continue } // failed imports have no text155 let chunks = chunker.chunk(extracted)156 do {157 let vectors = try await embed(chunks, provider, document)158 contents.append(.init(document: document, extracted: nil, chunks: chunks, vectors: vectors))159 } catch is CancellationError { report(document, .cancelled); throw CancellationError() }160 }161 try store.replaceIndex(contents, collection: collection)162 for c in contents { report(c.document, .finished, c.chunks.count, c.chunks.count) }163 }164165 private func embed(_ chunks: [TextChunk], _ provider: any EmbeddingProvider, _ document: KnowledgeDocument) async throws -> [[Float]] {166 var vectors: [[Float]] = []; vectors.reserveCapacity(chunks.count)167 report(document, .embedding, 0, chunks.count)168 for start in stride(from: 0, to: chunks.count, by: batchSize) {169 try Task.checkCancellation()170 let batch = chunks[start..<min(start + batchSize, chunks.count)].map(\.embeddingText)171 let result: [[Float]]172 do { result = try await provider.embed(batch) }173 catch is CancellationError { throw CancellationError() }174 catch { throw KnowledgeIndexingError.embeddingFailed((error as? LocalizedError)?.errorDescription ?? String(describing: error)) }175 guard result.count == batch.count else { throw KnowledgeStoreError.vectorCountMismatch(chunks: batch.count, vectors: result.count) }176 if let bad = result.first(where: { $0.count != provider.dimension }) { throw KnowledgeStoreError.dimensionMismatch(expected: provider.dimension, got: bad.count) }177 vectors.append(contentsOf: result)178 report(document, .embedding, vectors.count, chunks.count)179 }180 try Task.checkCancellation()181 return vectors182 }183184 private static func checkPrivacy(_ provider: any EmbeddingProvider, _ allowed: Bool) throws {185 if provider.isRemote && !allowed { throw KnowledgeIndexingError.remoteProcessingNotAllowed(provider.identifier) }186 }187}188