SPB Git

Repository Public

TextExtractor.swift 20.3 KB 319 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatKnowledge/Sources/TextExtractor.swift3// Description: On-device text extraction for PDF (optional Vision OCR), Markdown, plain text, DOCX, CSV and source code, with page / heading metadata.4import CoreGraphics5import Foundation6import PDFKit7import Vision89public enum DocumentKind: String, Sendable, Codable, CaseIterable { case pdf, md, txt, docx, csv, code }1011public struct ExtractedDocument: Sendable, Hashable {12    public struct Section: Sendable, Hashable {13        public var text: String14        public var page: Int?          // 1-based (PDF)15        public var heading: String?    // heading path, e.g. "Guide › Install"16        public init(text: String, page: Int? = nil, heading: String? = nil) { self.text = text; self.page = page; self.heading = heading }17    }18    public var kind: DocumentKind19    public var language: String?       // source code: language tag20    public var sections: [Section]21    public var characterCount: Int22    public var warnings: [String]23    public init(kind: DocumentKind, language: String? = nil, sections: [Section], warnings: [String] = []) {24        self.kind = kind; self.language = language; self.sections = sections; self.warnings = warnings25        characterCount = sections.reduce(0) { $0 + $1.text.count }26    }27}2829public enum ExtractionError: LocalizedError, Sendable, Equatable {30    case unsupported(String)31    case tooLarge(bytes: Int, limit: Int)32    case encrypted33    case corrupt(String)34    case empty(String)35    public var errorDescription: String? {36        switch self {37        case .unsupported(let m): "Unsupported file: \(m)"38        case .tooLarge(let b, let l): "The file is too large (\(b / 1_048_576) MB; the limit is \(l / 1_048_576) MB)."39        case .encrypted: "The file is password-protected and cannot be read."40        case .corrupt(let m): "The file is damaged or not what its extension says: \(m)"41        case .empty(let m): "No text could be extracted: \(m)"42        }43    }44}4546public struct TextExtractor: Sendable {47    public struct Options: Sendable {48        public var maxBytes: Int49        public var ocrPagesWithoutText: Bool    // Vision text recognition on PDF pages that have no text layer50        public var ocrLanguages: [String]51        public var maxPages: Int52        public init(maxBytes: Int = 200 * 1_048_576, ocrPagesWithoutText: Bool = false, ocrLanguages: [String] = [], maxPages: Int = 5000) {53            self.maxBytes = maxBytes; self.ocrPagesWithoutText = ocrPagesWithoutText; self.ocrLanguages = ocrLanguages; self.maxPages = maxPages54        }55    }56    public var options: Options57    public init(options: Options = .init()) { self.options = options }5859    static let codeLanguages: [String: String] = [60        "swift": "swift", "py": "python", "js": "javascript", "mjs": "javascript", "cjs": "javascript", "jsx": "jsx", "ts": "typescript", "tsx": "tsx",61        "c": "c", "h": "c", "m": "objective-c", "mm": "objective-c++", "cc": "cpp", "cpp": "cpp", "cxx": "cpp", "hpp": "cpp", "hh": "cpp",62        "rs": "rust", "go": "go", "java": "java", "kt": "kotlin", "kts": "kotlin", "scala": "scala", "rb": "ruby", "php": "php", "cs": "csharp",63        "fs": "fsharp", "r": "r", "jl": "julia", "lua": "lua", "pl": "perl", "sh": "bash", "bash": "bash", "zsh": "zsh", "fish": "fish",64        "ps1": "powershell", "sql": "sql", "html": "html", "htm": "html", "css": "css", "scss": "scss", "less": "less", "xml": "xml",65        "json": "json", "jsonl": "json", "yaml": "yaml", "yml": "yaml", "toml": "toml", "ini": "ini", "cfg": "ini", "conf": "ini",66        "tex": "latex", "bib": "bibtex", "dart": "dart", "ex": "elixir", "exs": "elixir", "erl": "erlang", "hs": "haskell", "ml": "ocaml",67        "clj": "clojure", "vue": "vue", "svelte": "svelte", "zig": "zig", "nim": "nim", "metal": "metal", "glsl": "glsl", "proto": "protobuf",68        "graphql": "graphql", "tf": "terraform", "gradle": "groovy", "groovy": "groovy", "cmake": "cmake", "mk": "makefile", "plist": "xml",69        "ipynb": "json", "vb": "vb", "asm": "asm", "s": "asm", "v": "verilog", "sv": "systemverilog", "vhd": "vhdl",70    ]71    static let codeFileNames: [String: String] = ["makefile": "makefile", "dockerfile": "dockerfile", "cmakelists.txt": "cmake", "gemfile": "ruby", "rakefile": "ruby", "podfile": "ruby"]7273    /// The kind implied by the file name, nil when the extension is unknown.74    public static func kind(forFileName name: String) -> (kind: DocumentKind, language: String?)? {75        let lower = (name as NSString).lastPathComponent.lowercased()76        if let l = codeFileNames[lower] { return (.code, l) }77        switch (lower as NSString).pathExtension {78        case "pdf": return (.pdf, nil)79        case "md", "markdown", "mdx", "mdown": return (.md, nil)80        case "txt", "text", "log", "rst", "org": return (.txt, nil)81        case "docx", "docm": return (.docx, nil)82        case "csv", "tsv", "tab": return (.csv, nil)83        case let ext: if let l = codeLanguages[ext] { return (.code, l) }; return nil84        }85    }86    public static var supportedExtensions: [String] { (["pdf", "md", "markdown", "mdx", "txt", "text", "log", "rst", "docx", "csv", "tsv"] + codeLanguages.keys).sorted() }8788    public func extract(fileURL: URL) throws -> ExtractedDocument {89        if let size = try? fileURL.resourceValues(forKeys: [.fileSizeKey]).fileSize, size > options.maxBytes { throw ExtractionError.tooLarge(bytes: size, limit: options.maxBytes) }90        let data: Data91        do { data = try Data(contentsOf: fileURL, options: .mappedIfSafe) } catch { throw ExtractionError.corrupt("cannot read the file (\(error.localizedDescription))") }92        return try extract(data: data, fileName: fileURL.lastPathComponent)93    }9495    public func extract(data: Data, fileName: String) throws -> ExtractedDocument {96        guard data.count <= options.maxBytes else { throw ExtractionError.tooLarge(bytes: data.count, limit: options.maxBytes) }97        guard !data.isEmpty else { throw ExtractionError.empty("the file is empty.") }98        let document: ExtractedDocument99        if let (kind, language) = Self.kind(forFileName: fileName) {100            switch kind {101            case .pdf: document = try pdf(data)102            case .docx: document = try docx(data)103            case .md: let (t, w) = try Self.decodeText(data); document = ExtractedDocument(kind: .md, sections: Self.markdownSections(t), warnings: w)104            case .txt: let (t, w) = try Self.decodeText(data); document = ExtractedDocument(kind: .txt, sections: [.init(text: Self.clean(t))], warnings: w)105            case .csv: document = try csv(data, fileName: fileName)106            case .code:107                let (t, w) = try Self.decodeText(data)108                document = ExtractedDocument(kind: .code, language: language, sections: [.init(text: t.replacingOccurrences(of: "\r\n", with: "\n"))], warnings: w)109            }110        } else if data.starts(with: [0x25, 0x50, 0x44, 0x46]) {   // "%PDF" without the extension111            document = try pdf(data)112        } else {113            // unknown extension: accept it only when it is clearly text114            let ext = (fileName as NSString).pathExtension115            guard let text = String(data: data, encoding: .utf8), !text.unicodeScalars.contains(where: { $0.value == 0 }) else {116                throw ExtractionError.unsupported(ext.isEmpty ? "\(fileName) is not a text file." : ".\(ext) files are not supported.")117            }118            document = ExtractedDocument(kind: .txt, sections: [.init(text: Self.clean(text))], warnings: ["Unknown file type, read as plain text."])119        }120        let kept = document.sections.filter { !$0.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }121        guard !kept.isEmpty else { throw ExtractionError.empty(document.warnings.first ?? "the document contains no text.") }122        return ExtractedDocument(kind: document.kind, language: document.language, sections: kept, warnings: document.warnings)123    }124125    // MARK: Plain text126127    static func clean(_ text: String) -> String {128        text.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n").replacingOccurrences(of: "\u{0}", with: "")129    }130131    /// UTF-8 (with or without BOM), UTF-16 (BOM or NUL-pattern heuristic), then Windows-1252 / Latin-1.132    static func decodeText(_ data: Data) throws -> (String, [String]) {133        func strip(_ s: String) -> String { s.hasPrefix("\u{FEFF}") ? String(s.dropFirst()) : s }134        if data.starts(with: [0xEF, 0xBB, 0xBF]), let s = String(data: data.dropFirst(3), encoding: .utf8) { return (s, []) }135        if data.starts(with: [0xFF, 0xFE]) || data.starts(with: [0xFE, 0xFF]), let s = String(data: data, encoding: .utf16) { return (strip(s), []) }136        let sample = data.prefix(4096); var evenZero = 0, oddZero = 0137        for (i, b) in sample.enumerated() where b == 0 { if i % 2 == 0 { evenZero += 1 } else { oddZero += 1 } }138        let half = max(1, sample.count / 2)139        if oddZero * 10 > half * 3, evenZero * 10 < half, let s = String(data: data, encoding: .utf16LittleEndian) { return (strip(s), ["Decoded as UTF-16 (little endian)."]) }140        if evenZero * 10 > half * 3, oddZero * 10 < half, let s = String(data: data, encoding: .utf16BigEndian) { return (strip(s), ["Decoded as UTF-16 (big endian)."]) }141        if evenZero + oddZero > sample.count / 20 { throw ExtractionError.unsupported("this looks like a binary file, not text.") }142        if let s = String(data: data, encoding: .utf8) { return (s, []) }143        if let s = String(data: data, encoding: .windowsCP1252) { return (s, ["Not valid UTF-8; decoded as Windows-1252."]) }144        return (String(data: data, encoding: .isoLatin1) ?? String(decoding: data, as: UTF8.self), ["Not valid UTF-8; decoded as Latin-1."])145    }146147    // MARK: Markdown148149    static func markdownSections(_ raw: String) -> [ExtractedDocument.Section] {150        var sections: [ExtractedDocument.Section] = []; var path: [(Int, String)] = []151        var lines: [Substring] = []; var headingLine: String?; var fence: String?152        func flush() {153            let body = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)154            if !body.isEmpty {155                let heading = path.isEmpty ? nil : path.map(\.1).joined(separator: " › ")156                sections.append(.init(text: (headingLine.map { $0 + "\n\n" } ?? "") + body, heading: heading))157            }158            lines = []159        }160        for line in clean(raw).split(separator: "\n", omittingEmptySubsequences: false) {161            let trimmed = line.drop(while: { $0 == " " })162            if let f = fence { lines.append(line); if trimmed.hasPrefix(f) { fence = nil }; continue }163            if trimmed.hasPrefix("```") || trimmed.hasPrefix("~~~") { fence = String(trimmed.prefix(3)); lines.append(line); continue }164            let hashes = trimmed.prefix(while: { $0 == "#" }).count165            if (1...6).contains(hashes), line.count - trimmed.count < 4, trimmed.dropFirst(hashes).first.map({ $0 == " " || $0 == "\t" }) == true {166                flush()167                var title = trimmed.dropFirst(hashes).trimmingCharacters(in: .whitespaces)168                while title.hasSuffix("#") { title = String(title.dropLast()) }169                title = title.trimmingCharacters(in: .whitespaces)170                while let last = path.last, last.0 >= hashes { path.removeLast() }171                path.append((hashes, title)); headingLine = String(trimmed)172            } else { lines.append(line) }173        }174        flush()175        return sections176    }177178    // MARK: CSV179180    func csv(_ data: Data, fileName: String) throws -> ExtractedDocument {181        var (text, warnings) = try Self.decodeText(data)182        text = Self.clean(text)183        let ext = (fileName as NSString).pathExtension.lowercased()184        let delimiter: Character = ext == "tsv" || ext == "tab" ? "\t" : DelimitedText.sniffDelimiter(text)185        let (rows, open) = DelimitedText.parse(text, delimiter: delimiter)186        if open { warnings.append("A quoted field is never closed; the end of the file may be misread.") }187        guard !rows.isEmpty else { throw ExtractionError.empty("the table has no rows.") }188        let widths = Set(rows.map(\.count))189        if widths.count > 1 { warnings.append("Rows have different numbers of columns (\(widths.min() ?? 0)–\(widths.max() ?? 0)).") }190        return ExtractedDocument(kind: .csv, sections: [.init(text: DelimitedText.render(rows: rows))], warnings: warnings)191    }192193    // MARK: DOCX194195    func docx(_ data: Data) throws -> ExtractedDocument {196        if data.starts(with: [0xD0, 0xCF, 0x11, 0xE0]) {197            // OLE container: either a password-protected OOXML file or a legacy .doc198            throw ExtractionError.encrypted199        }200        let xml: Data201        do {202            let zip = try ZipArchive(data: data)203            guard zip.entries["word/document.xml"] != nil else { throw ExtractionError.corrupt("word/document.xml is missing (not a Word document).") }204            xml = try zip.contents(of: "word/document.xml", maxSize: options.maxBytes * 8)205        } catch let e as ZipArchive.ZipError {206            switch e {207            case .encrypted: throw ExtractionError.encrypted208            case .notZip: throw ExtractionError.corrupt("not a DOCX (ZIP) file.")209            case .corrupt(let m), .unsupported(let m): throw ExtractionError.corrupt(m)210            case .missing(let m): throw ExtractionError.corrupt("\(m) is missing.")211            case .tooLarge: throw ExtractionError.tooLarge(bytes: data.count, limit: options.maxBytes)212            }213        }214        let delegate = DocxParser()215        let parser = XMLParser(data: xml); parser.delegate = delegate216        guard parser.parse() else { throw ExtractionError.corrupt("invalid XML in the document (\(parser.parserError?.localizedDescription ?? "unknown error")).") }217        var sections: [ExtractedDocument.Section] = []; var heading: String?; var parts: [String] = []; var lastWasRow = false218        func flush() {219            let body = parts.joined().trimmingCharacters(in: .whitespacesAndNewlines)220            if !body.isEmpty { sections.append(.init(text: body, heading: heading)) }221            parts = []; lastWasRow = false222        }223        for block in delegate.blocks {224            let text = block.text.trimmingCharacters(in: .whitespaces)225            if text.isEmpty { continue }226            if block.headingLevel != nil { flush(); heading = text; parts.append(text); continue }227            parts.append((parts.isEmpty ? "" : (block.isRow && lastWasRow ? "\n" : "\n\n")) + text)228            lastWasRow = block.isRow229        }230        flush()231        return ExtractedDocument(kind: .docx, sections: sections)232    }233234    // MARK: PDF235236    func pdf(_ data: Data) throws -> ExtractedDocument {237        guard data.prefix(1024).range(of: Data("%PDF".utf8)) != nil, let document = PDFDocument(data: data) else { throw ExtractionError.corrupt("not a readable PDF.") }238        if document.isLocked, !document.unlock(withPassword: "") { throw ExtractionError.encrypted }239        var sections: [ExtractedDocument.Section] = []; var warnings: [String] = []; var missing: [Int] = []; var recognized = 0240        let count = min(document.pageCount, options.maxPages)241        if document.pageCount > count { warnings.append("Only the first \(count) of \(document.pageCount) pages were read.") }242        for index in 0..<count {243            guard let page = document.page(at: index) else { continue }244            var text = (page.string ?? "").trimmingCharacters(in: .whitespacesAndNewlines)245            if text.isEmpty {246                if options.ocrPagesWithoutText, let ocr = Self.recognizeText(page: page, languages: options.ocrLanguages), !ocr.isEmpty { text = ocr; recognized += 1 }247                else { missing.append(index + 1); continue }248            }249            sections.append(.init(text: Self.clean(text), page: index + 1))250        }251        if recognized > 0 { warnings.append("\(recognized) page\(recognized == 1 ? "" : "s") had no text layer and were read with on-device OCR.") }252        if !missing.isEmpty {253            let list = missing.prefix(12).map(String.init).joined(separator: ", ") + (missing.count > 12 ? ", …" : "")254            warnings.insert("\(missing.count) page\(missing.count == 1 ? " has" : "s have") no text layer and need\(missing.count == 1 ? "s" : "") OCR (page\(missing.count == 1 ? "" : "s") \(list)).", at: 0)255        }256        return ExtractedDocument(kind: .pdf, sections: sections, warnings: warnings)257    }258259    /// Renders the page to a bitmap and runs Vision text recognition (on device).260    static func recognizeText(page: PDFPage, languages: [String]) -> String? {261        let bounds = page.bounds(for: .mediaBox)262        guard bounds.width > 1, bounds.height > 1 else { return nil }263        let scale = min(3.0, max(1.0, 2400 / max(bounds.width, bounds.height)))264        let width = Int(bounds.width * scale), height = Int(bounds.height * scale)265        guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),266                                      bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }267        context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1)); context.fill(CGRect(x: 0, y: 0, width: width, height: height))268        context.scaleBy(x: scale, y: scale); context.translateBy(x: -bounds.minX, y: -bounds.minY)269        page.draw(with: .mediaBox, to: context)270        guard let image = context.makeImage() else { return nil }271        let request = VNRecognizeTextRequest()272        request.recognitionLevel = .accurate; request.usesLanguageCorrection = true273        if !languages.isEmpty { request.recognitionLanguages = languages } else { request.automaticallyDetectsLanguage = true }274        guard (try? VNImageRequestHandler(cgImage: image, options: [:]).perform([request])) != nil, let results = request.results else { return nil }275        return results.compactMap { $0.topCandidates(1).first?.string }.joined(separator: "\n")276    }277}278279/// WordprocessingML → blocks (paragraphs, headings, table rows).280final class DocxParser: NSObject, XMLParserDelegate {281    struct Block { var text: String; var headingLevel: Int?; var isRow: Bool }282    var blocks: [Block] = []283    private var current = ""; private var inText = false; private var headingLevel: Int?284    private var tableDepth = 0; private var rowCells: [String] = []; private var cellParagraphs: [String] = []285286    func parser(_ parser: XMLParser, didStartElement name: String, namespaceURI: String?, qualifiedName: String?, attributes: [String: String] = [:]) {287        switch name {288        case "w:p": current = ""; headingLevel = nil289        case "w:pStyle":290            let value = (attributes["w:val"] ?? "").lowercased()291            if value == "title" || value == "titre" { headingLevel = 1 }292            else if ["heading", "titre", "berschrift", "ttulo", "titolo", "kop"].contains(where: value.contains) { headingLevel = Int(value.filter(\.isNumber)) ?? 1 }293        case "w:outlineLvl": if let v = attributes["w:val"].flatMap(Int.init), v < 9, headingLevel == nil { headingLevel = v + 1 }294        case "w:t": inText = true295        case "w:tab": current += "\t"296        case "w:br", "w:cr": current += "\n"297        case "w:noBreakHyphen": current += "-"298        case "w:tbl": tableDepth += 1299        case "w:tr": if tableDepth == 1 { rowCells = [] }300        case "w:tc": if tableDepth == 1 { cellParagraphs = [] }301        default: break302        }303    }304    func parser(_ parser: XMLParser, foundCharacters string: String) { if inText { current += string } }305    func parser(_ parser: XMLParser, didEndElement name: String, namespaceURI: String?, qualifiedName: String?) {306        switch name {307        case "w:t": inText = false308        case "w:p":309            if tableDepth > 0 { let t = current.trimmingCharacters(in: .whitespacesAndNewlines); if !t.isEmpty { cellParagraphs.append(t) } }310            else { blocks.append(Block(text: current, headingLevel: headingLevel, isRow: false)) }311            current = ""312        case "w:tc": if tableDepth == 1 { rowCells.append(cellParagraphs.joined(separator: " ")); cellParagraphs = [] }313        case "w:tr": if tableDepth == 1, rowCells.contains(where: { !$0.isEmpty }) { blocks.append(Block(text: rowCells.joined(separator: " | "), headingLevel: nil, isRow: true)) }314        case "w:tbl": tableDepth = max(0, tableDepth - 1)315        default: break316        }317    }318}319
Files main
Menu

Theme