SPB Git

Repository Public

KnowledgeIndexer.swift 12.2 KB 188 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatKnowledge/Sources/KnowledgeIndexer.swift3// Description: Document pipeline: hash → extract → chunk → embed in batches → one transactional write. Deduplicates, reports progress, cancels cleanly, re-indexes.4import CryptoKit5import Foundation6import LocalChatCore78public struct IndexingProgress: Sendable, Hashable {9    public enum Stage: String, Sendable { case extracting, chunking, embedding, writing, finished, duplicate, failed, cancelled }10    public var documentID: UUID11    public var documentName: String12    public var collectionID: UUID13    public var stage: Stage14    public var chunksDone: Int15    public var chunksTotal: Int16    public var message: String?17    public var fraction: Double? { chunksTotal > 0 ? Double(chunksDone) / Double(chunksTotal) : nil }18}1920public enum KnowledgeIndexingError: LocalizedError, Sendable, Equatable {21    case remoteProcessingNotAllowed(String)22    case embedderMismatch(collection: String, provider: String)23    case embeddingFailed(String)24    case nothingToIndex25    public var errorDescription: String? {26        switch self {27        case .remoteProcessingNotAllowed(let p): "“\(p)” runs on another machine. Your documents stay on this device unless you explicitly allow remote processing."28        case .embedderMismatch(let c, let p): "This collection was indexed with “\(c)”. Re-index the collection to switch to “\(p)”; vectors from two models cannot be mixed."29        case .embeddingFailed(let m): "The embedding model failed: \(m)"30        case .nothingToIndex: "The document produced no text to index."31        }32    }33}3435public actor KnowledgeIndexer {36    public let store: KnowledgeStore37    public let extractor: TextExtractor38    public let batchSize: Int39    private var observers: [UUID: AsyncStream<IndexingProgress>.Continuation] = [:]40    private var busyHashes: Set<String> = []4142    public init(store: KnowledgeStore, extractor: TextExtractor = .init(), batchSize: Int = 32) {43        self.store = store; self.extractor = extractor; self.batchSize = max(1, batchSize)44    }4546    /// Every progress event of this indexer, for as long as the returned stream is iterated.47    public func progressUpdates() -> AsyncStream<IndexingProgress> {48        let id = UUID()49        let (stream, continuation) = AsyncStream<IndexingProgress>.makeStream(bufferingPolicy: .bufferingNewest(512))50        observers[id] = continuation51        continuation.onTermination = { [weak self] _ in Task { await self?.removeObserver(id) } }52        return stream53    }54    private func removeObserver(_ id: UUID) { observers[id] = nil }55    private func report(_ d: KnowledgeDocument, _ stage: IndexingProgress.Stage, _ done: Int = 0, _ total: Int = 0, _ message: String? = nil) {56        let event = IndexingProgress(documentID: d.id, documentName: d.name, collectionID: d.collectionID, stage: stage, chunksDone: done, chunksTotal: total, message: message)57        for c in observers.values { c.yield(event) }58    }5960    @discardableResult61    public func add(fileURL: URL, to collectionID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws -> KnowledgeDocument {62        let scoped = fileURL.startAccessingSecurityScopedResource()63        defer { if scoped { fileURL.stopAccessingSecurityScopedResource() } }64        if let size = try? fileURL.resourceValues(forKeys: [.fileSizeKey]).fileSize, size > extractor.options.maxBytes {65            throw ExtractionError.tooLarge(bytes: size, limit: extractor.options.maxBytes)66        }67        let data: Data68        do { data = try Data(contentsOf: fileURL, options: .mappedIfSafe) } catch { throw ExtractionError.corrupt("cannot read the file (\(error.localizedDescription))") }69        return try await add(data: data, name: fileURL.lastPathComponent, to: collectionID, provider: provider, allowRemoteProcessing: allowRemoteProcessing)70    }7172    /// Adds a document. An identical file (same SHA-256) already indexed in the collection is returned as is.73    @discardableResult74    public func add(data: Data, name: String, to collectionID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws -> KnowledgeDocument {75        try Self.checkPrivacy(provider, allowRemoteProcessing)76        guard var collection = try store.collection(collectionID) else { throw KnowledgeStoreError.collectionNotFound }77        let hash = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined()78        let busyKey = collectionID.uuidString + hash79        var document: KnowledgeDocument80        if let existing = try store.document(inCollection: collectionID, contentHash: hash) {81            if existing.status != .failed || busyHashes.contains(busyKey) { report(existing, .duplicate, existing.chunkCount, existing.chunkCount, "Already in this collection."); return existing }82            document = existing; document.name = name      // retry a failed import in place83        } else {84            document = KnowledgeDocument(collectionID: collectionID, name: name, kind: TextExtractor.kind(forFileName: name)?.kind.rawValue ?? "txt", byteCount: data.count, contentHash: hash)85        }86        // vectors of two embedding models are never mixed: an empty collection adopts the provider, otherwise it must match87        if collection.embedderID != provider.identifier || collection.dimension != provider.dimension {88            guard try store.counts(inCollection: collectionID).vectors == 0 else { throw KnowledgeIndexingError.embedderMismatch(collection: collection.embedderID, provider: provider.identifier) }89            collection.embedderID = provider.identifier; collection.dimension = provider.dimension; collection.updatedAt = Date()90            try store.saveCollection(collection)91        }92        document.status = .indexing; document.errorText = nil93        try store.saveDocument(document)            // visible to concurrent adds of the same file94        busyHashes.insert(busyKey); defer { busyHashes.remove(busyKey) }95        do {96            report(document, .extracting)97            let extractor = extractor98            let extracted = try await Task.detached(priority: .userInitiated) { try extractor.extract(data: data, fileName: name) }.value99            document.kind = extracted.kind.rawValue; document.characterCount = extracted.characterCount100            try Task.checkCancellation()101            report(document, .chunking)102            let chunks = Chunker(collection: collection).chunk(extracted)103            guard !chunks.isEmpty else { throw KnowledgeIndexingError.nothingToIndex }104            let vectors = try await embed(chunks, provider, document)105            report(document, .writing, chunks.count, chunks.count)106            try store.replaceIndex([.init(document: document, extracted: extracted, chunks: chunks, vectors: vectors)])107            let saved = try store.document(document.id) ?? document108            report(saved, .finished, chunks.count, chunks.count, extracted.warnings.first)109            return saved110        } catch is CancellationError {111            try? store.deleteDocument(document.id)     // nothing partial is left behind112            report(document, .cancelled)113            throw CancellationError()114        } catch {115            document.status = .failed; document.chunkCount = 0116            document.errorText = (error as? LocalizedError)?.errorDescription ?? String(describing: error)117            try? store.saveDocument(document)118            report(document, .failed, 0, 0, document.errorText)119            throw error120        }121    }122123    /// Re-chunks (with the collection's current settings) and re-embeds one document from its stored text. The previous index stays in place until the new one is written.124    @discardableResult125    public func reindex(documentID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws -> KnowledgeDocument {126        try Self.checkPrivacy(provider, allowRemoteProcessing)127        guard let document = try store.document(documentID) else { throw KnowledgeStoreError.documentNotFound }128        guard let collection = try store.collection(document.collectionID) else { throw KnowledgeStoreError.collectionNotFound }129        guard collection.embedderID == provider.identifier, collection.dimension == provider.dimension else {130            throw KnowledgeIndexingError.embedderMismatch(collection: collection.embedderID, provider: provider.identifier)131        }132        guard let extracted = try store.extractedDocument(documentID), !extracted.sections.isEmpty else { throw KnowledgeIndexingError.nothingToIndex }133        do {134            let chunks = Chunker(collection: collection).chunk(extracted)135            guard !chunks.isEmpty else { throw KnowledgeIndexingError.nothingToIndex }136            let vectors = try await embed(chunks, provider, document)137            report(document, .writing, chunks.count, chunks.count)138            try store.replaceIndex([.init(document: document, extracted: nil, chunks: chunks, vectors: vectors)])139            let saved = try store.document(documentID) ?? document140            report(saved, .finished, chunks.count, chunks.count)141            return saved142        } catch is CancellationError { report(document, .cancelled); throw CancellationError() }143    }144145    /// Re-indexes every document of the collection with `provider` and adopts it (identifier, dimension) in one transaction:146    /// the collection never holds vectors of two models, and a cancellation or failure leaves the previous index untouched.147    public func reindex(collectionID: UUID, provider: any EmbeddingProvider, allowRemoteProcessing: Bool = false) async throws {148        try Self.checkPrivacy(provider, allowRemoteProcessing)149        guard var collection = try store.collection(collectionID) else { throw KnowledgeStoreError.collectionNotFound }150        collection.embedderID = provider.identifier; collection.dimension = provider.dimension; collection.updatedAt = Date()151        var contents: [KnowledgeStore.IndexedContent] = []152        let chunker = Chunker(collection: collection)153        for document in try store.documents(in: collectionID) {154            guard let extracted = try store.extractedDocument(document.id), !extracted.sections.isEmpty else { continue }   // failed imports have no text155            let chunks = chunker.chunk(extracted)156            do {157                let vectors = try await embed(chunks, provider, document)158                contents.append(.init(document: document, extracted: nil, chunks: chunks, vectors: vectors))159            } catch is CancellationError { report(document, .cancelled); throw CancellationError() }160        }161        try store.replaceIndex(contents, collection: collection)162        for c in contents { report(c.document, .finished, c.chunks.count, c.chunks.count) }163    }164165    private func embed(_ chunks: [TextChunk], _ provider: any EmbeddingProvider, _ document: KnowledgeDocument) async throws -> [[Float]] {166        var vectors: [[Float]] = []; vectors.reserveCapacity(chunks.count)167        report(document, .embedding, 0, chunks.count)168        for start in stride(from: 0, to: chunks.count, by: batchSize) {169            try Task.checkCancellation()170            let batch = chunks[start..<min(start + batchSize, chunks.count)].map(\.embeddingText)171            let result: [[Float]]172            do { result = try await provider.embed(batch) }173            catch is CancellationError { throw CancellationError() }174            catch { throw KnowledgeIndexingError.embeddingFailed((error as? LocalizedError)?.errorDescription ?? String(describing: error)) }175            guard result.count == batch.count else { throw KnowledgeStoreError.vectorCountMismatch(chunks: batch.count, vectors: result.count) }176            if let bad = result.first(where: { $0.count != provider.dimension }) { throw KnowledgeStoreError.dimensionMismatch(expected: provider.dimension, got: bad.count) }177            vectors.append(contentsOf: result)178            report(document, .embedding, vectors.count, chunks.count)179        }180        try Task.checkCancellation()181        return vectors182    }183184    private static func checkPrivacy(_ provider: any EmbeddingProvider, _ allowed: Bool) throws {185        if provider.isRemote && !allowed { throw KnowledgeIndexingError.remoteProcessingNotAllowed(provider.identifier) }186    }187}188
Files main
Menu

Theme