Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatKnowledge/Sources/TextExtractor.swift3// Description: On-device text extraction for PDF (optional Vision OCR), Markdown, plain text, DOCX, CSV and source code, with page / heading metadata.4import CoreGraphics5import Foundation6import PDFKit7import Vision89public enum DocumentKind: String, Sendable, Codable, CaseIterable { case pdf, md, txt, docx, csv, code }1011public struct ExtractedDocument: Sendable, Hashable {12 public struct Section: Sendable, Hashable {13 public var text: String14 public var page: Int? // 1-based (PDF)15 public var heading: String? // heading path, e.g. "Guide › Install"16 public init(text: String, page: Int? = nil, heading: String? = nil) { self.text = text; self.page = page; self.heading = heading }17 }18 public var kind: DocumentKind19 public var language: String? // source code: language tag20 public var sections: [Section]21 public var characterCount: Int22 public var warnings: [String]23 public init(kind: DocumentKind, language: String? = nil, sections: [Section], warnings: [String] = []) {24 self.kind = kind; self.language = language; self.sections = sections; self.warnings = warnings25 characterCount = sections.reduce(0) { $0 + $1.text.count }26 }27}2829public enum ExtractionError: LocalizedError, Sendable, Equatable {30 case unsupported(String)31 case tooLarge(bytes: Int, limit: Int)32 case encrypted33 case corrupt(String)34 case empty(String)35 public var errorDescription: String? {36 switch self {37 case .unsupported(let m): "Unsupported file: \(m)"38 case .tooLarge(let b, let l): "The file is too large (\(b / 1_048_576) MB; the limit is \(l / 1_048_576) MB)."39 case .encrypted: "The file is password-protected and cannot be read."40 case .corrupt(let m): "The file is damaged or not what its extension says: \(m)"41 case .empty(let m): "No text could be extracted: \(m)"42 }43 }44}4546public struct TextExtractor: Sendable {47 public struct Options: Sendable {48 public var maxBytes: Int49 public var ocrPagesWithoutText: Bool // Vision text recognition on PDF pages that have no text layer50 public var ocrLanguages: [String]51 public var maxPages: Int52 public init(maxBytes: Int = 200 * 1_048_576, ocrPagesWithoutText: Bool = false, ocrLanguages: [String] = [], maxPages: Int = 5000) {53 self.maxBytes = maxBytes; self.ocrPagesWithoutText = ocrPagesWithoutText; self.ocrLanguages = ocrLanguages; self.maxPages = maxPages54 }55 }56 public var options: Options57 public init(options: Options = .init()) { self.options = options }5859 static let codeLanguages: [String: String] = [60 "swift": "swift", "py": "python", "js": "javascript", "mjs": "javascript", "cjs": "javascript", "jsx": "jsx", "ts": "typescript", "tsx": "tsx",61 "c": "c", "h": "c", "m": "objective-c", "mm": "objective-c++", "cc": "cpp", "cpp": "cpp", "cxx": "cpp", "hpp": "cpp", "hh": "cpp",62 "rs": "rust", "go": "go", "java": "java", "kt": "kotlin", "kts": "kotlin", "scala": "scala", "rb": "ruby", "php": "php", "cs": "csharp",63 "fs": "fsharp", "r": "r", "jl": "julia", "lua": "lua", "pl": "perl", "sh": "bash", "bash": "bash", "zsh": "zsh", "fish": "fish",64 "ps1": "powershell", "sql": "sql", "html": "html", "htm": "html", "css": "css", "scss": "scss", "less": "less", "xml": "xml",65 "json": "json", "jsonl": "json", "yaml": "yaml", "yml": "yaml", "toml": "toml", "ini": "ini", "cfg": "ini", "conf": "ini",66 "tex": "latex", "bib": "bibtex", "dart": "dart", "ex": "elixir", "exs": "elixir", "erl": "erlang", "hs": "haskell", "ml": "ocaml",67 "clj": "clojure", "vue": "vue", "svelte": "svelte", "zig": "zig", "nim": "nim", "metal": "metal", "glsl": "glsl", "proto": "protobuf",68 "graphql": "graphql", "tf": "terraform", "gradle": "groovy", "groovy": "groovy", "cmake": "cmake", "mk": "makefile", "plist": "xml",69 "ipynb": "json", "vb": "vb", "asm": "asm", "s": "asm", "v": "verilog", "sv": "systemverilog", "vhd": "vhdl",70 ]71 static let codeFileNames: [String: String] = ["makefile": "makefile", "dockerfile": "dockerfile", "cmakelists.txt": "cmake", "gemfile": "ruby", "rakefile": "ruby", "podfile": "ruby"]7273 /// The kind implied by the file name, nil when the extension is unknown.74 public static func kind(forFileName name: String) -> (kind: DocumentKind, language: String?)? {75 let lower = (name as NSString).lastPathComponent.lowercased()76 if let l = codeFileNames[lower] { return (.code, l) }77 switch (lower as NSString).pathExtension {78 case "pdf": return (.pdf, nil)79 case "md", "markdown", "mdx", "mdown": return (.md, nil)80 case "txt", "text", "log", "rst", "org": return (.txt, nil)81 case "docx", "docm": return (.docx, nil)82 case "csv", "tsv", "tab": return (.csv, nil)83 case let ext: if let l = codeLanguages[ext] { return (.code, l) }; return nil84 }85 }86 public static var supportedExtensions: [String] { (["pdf", "md", "markdown", "mdx", "txt", "text", "log", "rst", "docx", "csv", "tsv"] + codeLanguages.keys).sorted() }8788 public func extract(fileURL: URL) throws -> ExtractedDocument {89 if let size = try? fileURL.resourceValues(forKeys: [.fileSizeKey]).fileSize, size > options.maxBytes { throw ExtractionError.tooLarge(bytes: size, limit: options.maxBytes) }90 let data: Data91 do { data = try Data(contentsOf: fileURL, options: .mappedIfSafe) } catch { throw ExtractionError.corrupt("cannot read the file (\(error.localizedDescription))") }92 return try extract(data: data, fileName: fileURL.lastPathComponent)93 }9495 public func extract(data: Data, fileName: String) throws -> ExtractedDocument {96 guard data.count <= options.maxBytes else { throw ExtractionError.tooLarge(bytes: data.count, limit: options.maxBytes) }97 guard !data.isEmpty else { throw ExtractionError.empty("the file is empty.") }98 let document: ExtractedDocument99 if let (kind, language) = Self.kind(forFileName: fileName) {100 switch kind {101 case .pdf: document = try pdf(data)102 case .docx: document = try docx(data)103 case .md: let (t, w) = try Self.decodeText(data); document = ExtractedDocument(kind: .md, sections: Self.markdownSections(t), warnings: w)104 case .txt: let (t, w) = try Self.decodeText(data); document = ExtractedDocument(kind: .txt, sections: [.init(text: Self.clean(t))], warnings: w)105 case .csv: document = try csv(data, fileName: fileName)106 case .code:107 let (t, w) = try Self.decodeText(data)108 document = ExtractedDocument(kind: .code, language: language, sections: [.init(text: t.replacingOccurrences(of: "\r\n", with: "\n"))], warnings: w)109 }110 } else if data.starts(with: [0x25, 0x50, 0x44, 0x46]) { // "%PDF" without the extension111 document = try pdf(data)112 } else {113 // unknown extension: accept it only when it is clearly text114 let ext = (fileName as NSString).pathExtension115 guard let text = String(data: data, encoding: .utf8), !text.unicodeScalars.contains(where: { $0.value == 0 }) else {116 throw ExtractionError.unsupported(ext.isEmpty ? "\(fileName) is not a text file." : ".\(ext) files are not supported.")117 }118 document = ExtractedDocument(kind: .txt, sections: [.init(text: Self.clean(text))], warnings: ["Unknown file type, read as plain text."])119 }120 let kept = document.sections.filter { !$0.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }121 guard !kept.isEmpty else { throw ExtractionError.empty(document.warnings.first ?? "the document contains no text.") }122 return ExtractedDocument(kind: document.kind, language: document.language, sections: kept, warnings: document.warnings)123 }124125 // MARK: Plain text126127 static func clean(_ text: String) -> String {128 text.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n").replacingOccurrences(of: "\u{0}", with: "")129 }130131 /// UTF-8 (with or without BOM), UTF-16 (BOM or NUL-pattern heuristic), then Windows-1252 / Latin-1.132 static func decodeText(_ data: Data) throws -> (String, [String]) {133 func strip(_ s: String) -> String { s.hasPrefix("\u{FEFF}") ? String(s.dropFirst()) : s }134 if data.starts(with: [0xEF, 0xBB, 0xBF]), let s = String(data: data.dropFirst(3), encoding: .utf8) { return (s, []) }135 if data.starts(with: [0xFF, 0xFE]) || data.starts(with: [0xFE, 0xFF]), let s = String(data: data, encoding: .utf16) { return (strip(s), []) }136 let sample = data.prefix(4096); var evenZero = 0, oddZero = 0137 for (i, b) in sample.enumerated() where b == 0 { if i % 2 == 0 { evenZero += 1 } else { oddZero += 1 } }138 let half = max(1, sample.count / 2)139 if oddZero * 10 > half * 3, evenZero * 10 < half, let s = String(data: data, encoding: .utf16LittleEndian) { return (strip(s), ["Decoded as UTF-16 (little endian)."]) }140 if evenZero * 10 > half * 3, oddZero * 10 < half, let s = String(data: data, encoding: .utf16BigEndian) { return (strip(s), ["Decoded as UTF-16 (big endian)."]) }141 if evenZero + oddZero > sample.count / 20 { throw ExtractionError.unsupported("this looks like a binary file, not text.") }142 if let s = String(data: data, encoding: .utf8) { return (s, []) }143 if let s = String(data: data, encoding: .windowsCP1252) { return (s, ["Not valid UTF-8; decoded as Windows-1252."]) }144 return (String(data: data, encoding: .isoLatin1) ?? String(decoding: data, as: UTF8.self), ["Not valid UTF-8; decoded as Latin-1."])145 }146147 // MARK: Markdown148149 static func markdownSections(_ raw: String) -> [ExtractedDocument.Section] {150 var sections: [ExtractedDocument.Section] = []; var path: [(Int, String)] = []151 var lines: [Substring] = []; var headingLine: String?; var fence: String?152 func flush() {153 let body = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)154 if !body.isEmpty {155 let heading = path.isEmpty ? nil : path.map(\.1).joined(separator: " › ")156 sections.append(.init(text: (headingLine.map { $0 + "\n\n" } ?? "") + body, heading: heading))157 }158 lines = []159 }160 for line in clean(raw).split(separator: "\n", omittingEmptySubsequences: false) {161 let trimmed = line.drop(while: { $0 == " " })162 if let f = fence { lines.append(line); if trimmed.hasPrefix(f) { fence = nil }; continue }163 if trimmed.hasPrefix("```") || trimmed.hasPrefix("~~~") { fence = String(trimmed.prefix(3)); lines.append(line); continue }164 let hashes = trimmed.prefix(while: { $0 == "#" }).count165 if (1...6).contains(hashes), line.count - trimmed.count < 4, trimmed.dropFirst(hashes).first.map({ $0 == " " || $0 == "\t" }) == true {166 flush()167 var title = trimmed.dropFirst(hashes).trimmingCharacters(in: .whitespaces)168 while title.hasSuffix("#") { title = String(title.dropLast()) }169 title = title.trimmingCharacters(in: .whitespaces)170 while let last = path.last, last.0 >= hashes { path.removeLast() }171 path.append((hashes, title)); headingLine = String(trimmed)172 } else { lines.append(line) }173 }174 flush()175 return sections176 }177178 // MARK: CSV179180 func csv(_ data: Data, fileName: String) throws -> ExtractedDocument {181 var (text, warnings) = try Self.decodeText(data)182 text = Self.clean(text)183 let ext = (fileName as NSString).pathExtension.lowercased()184 let delimiter: Character = ext == "tsv" || ext == "tab" ? "\t" : DelimitedText.sniffDelimiter(text)185 let (rows, open) = DelimitedText.parse(text, delimiter: delimiter)186 if open { warnings.append("A quoted field is never closed; the end of the file may be misread.") }187 guard !rows.isEmpty else { throw ExtractionError.empty("the table has no rows.") }188 let widths = Set(rows.map(\.count))189 if widths.count > 1 { warnings.append("Rows have different numbers of columns (\(widths.min() ?? 0)–\(widths.max() ?? 0)).") }190 return ExtractedDocument(kind: .csv, sections: [.init(text: DelimitedText.render(rows: rows))], warnings: warnings)191 }192193 // MARK: DOCX194195 func docx(_ data: Data) throws -> ExtractedDocument {196 if data.starts(with: [0xD0, 0xCF, 0x11, 0xE0]) {197 // OLE container: either a password-protected OOXML file or a legacy .doc198 throw ExtractionError.encrypted199 }200 let xml: Data201 do {202 let zip = try ZipArchive(data: data)203 guard zip.entries["word/document.xml"] != nil else { throw ExtractionError.corrupt("word/document.xml is missing (not a Word document).") }204 xml = try zip.contents(of: "word/document.xml", maxSize: options.maxBytes * 8)205 } catch let e as ZipArchive.ZipError {206 switch e {207 case .encrypted: throw ExtractionError.encrypted208 case .notZip: throw ExtractionError.corrupt("not a DOCX (ZIP) file.")209 case .corrupt(let m), .unsupported(let m): throw ExtractionError.corrupt(m)210 case .missing(let m): throw ExtractionError.corrupt("\(m) is missing.")211 case .tooLarge: throw ExtractionError.tooLarge(bytes: data.count, limit: options.maxBytes)212 }213 }214 let delegate = DocxParser()215 let parser = XMLParser(data: xml); parser.delegate = delegate216 guard parser.parse() else { throw ExtractionError.corrupt("invalid XML in the document (\(parser.parserError?.localizedDescription ?? "unknown error")).") }217 var sections: [ExtractedDocument.Section] = []; var heading: String?; var parts: [String] = []; var lastWasRow = false218 func flush() {219 let body = parts.joined().trimmingCharacters(in: .whitespacesAndNewlines)220 if !body.isEmpty { sections.append(.init(text: body, heading: heading)) }221 parts = []; lastWasRow = false222 }223 for block in delegate.blocks {224 let text = block.text.trimmingCharacters(in: .whitespaces)225 if text.isEmpty { continue }226 if block.headingLevel != nil { flush(); heading = text; parts.append(text); continue }227 parts.append((parts.isEmpty ? "" : (block.isRow && lastWasRow ? "\n" : "\n\n")) + text)228 lastWasRow = block.isRow229 }230 flush()231 return ExtractedDocument(kind: .docx, sections: sections)232 }233234 // MARK: PDF235236 func pdf(_ data: Data) throws -> ExtractedDocument {237 guard data.prefix(1024).range(of: Data("%PDF".utf8)) != nil, let document = PDFDocument(data: data) else { throw ExtractionError.corrupt("not a readable PDF.") }238 if document.isLocked, !document.unlock(withPassword: "") { throw ExtractionError.encrypted }239 var sections: [ExtractedDocument.Section] = []; var warnings: [String] = []; var missing: [Int] = []; var recognized = 0240 let count = min(document.pageCount, options.maxPages)241 if document.pageCount > count { warnings.append("Only the first \(count) of \(document.pageCount) pages were read.") }242 for index in 0..<count {243 guard let page = document.page(at: index) else { continue }244 var text = (page.string ?? "").trimmingCharacters(in: .whitespacesAndNewlines)245 if text.isEmpty {246 if options.ocrPagesWithoutText, let ocr = Self.recognizeText(page: page, languages: options.ocrLanguages), !ocr.isEmpty { text = ocr; recognized += 1 }247 else { missing.append(index + 1); continue }248 }249 sections.append(.init(text: Self.clean(text), page: index + 1))250 }251 if recognized > 0 { warnings.append("\(recognized) page\(recognized == 1 ? "" : "s") had no text layer and were read with on-device OCR.") }252 if !missing.isEmpty {253 let list = missing.prefix(12).map(String.init).joined(separator: ", ") + (missing.count > 12 ? ", …" : "")254 warnings.insert("\(missing.count) page\(missing.count == 1 ? " has" : "s have") no text layer and need\(missing.count == 1 ? "s" : "") OCR (page\(missing.count == 1 ? "" : "s") \(list)).", at: 0)255 }256 return ExtractedDocument(kind: .pdf, sections: sections, warnings: warnings)257 }258259 /// Renders the page to a bitmap and runs Vision text recognition (on device).260 static func recognizeText(page: PDFPage, languages: [String]) -> String? {261 let bounds = page.bounds(for: .mediaBox)262 guard bounds.width > 1, bounds.height > 1 else { return nil }263 let scale = min(3.0, max(1.0, 2400 / max(bounds.width, bounds.height)))264 let width = Int(bounds.width * scale), height = Int(bounds.height * scale)265 guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),266 bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }267 context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1)); context.fill(CGRect(x: 0, y: 0, width: width, height: height))268 context.scaleBy(x: scale, y: scale); context.translateBy(x: -bounds.minX, y: -bounds.minY)269 page.draw(with: .mediaBox, to: context)270 guard let image = context.makeImage() else { return nil }271 let request = VNRecognizeTextRequest()272 request.recognitionLevel = .accurate; request.usesLanguageCorrection = true273 if !languages.isEmpty { request.recognitionLanguages = languages } else { request.automaticallyDetectsLanguage = true }274 guard (try? VNImageRequestHandler(cgImage: image, options: [:]).perform([request])) != nil, let results = request.results else { return nil }275 return results.compactMap { $0.topCandidates(1).first?.string }.joined(separator: "\n")276 }277}278279/// WordprocessingML → blocks (paragraphs, headings, table rows).280final class DocxParser: NSObject, XMLParserDelegate {281 struct Block { var text: String; var headingLevel: Int?; var isRow: Bool }282 var blocks: [Block] = []283 private var current = ""; private var inText = false; private var headingLevel: Int?284 private var tableDepth = 0; private var rowCells: [String] = []; private var cellParagraphs: [String] = []285286 func parser(_ parser: XMLParser, didStartElement name: String, namespaceURI: String?, qualifiedName: String?, attributes: [String: String] = [:]) {287 switch name {288 case "w:p": current = ""; headingLevel = nil289 case "w:pStyle":290 let value = (attributes["w:val"] ?? "").lowercased()291 if value == "title" || value == "titre" { headingLevel = 1 }292 else if ["heading", "titre", "berschrift", "ttulo", "titolo", "kop"].contains(where: value.contains) { headingLevel = Int(value.filter(\.isNumber)) ?? 1 }293 case "w:outlineLvl": if let v = attributes["w:val"].flatMap(Int.init), v < 9, headingLevel == nil { headingLevel = v + 1 }294 case "w:t": inText = true295 case "w:tab": current += "\t"296 case "w:br", "w:cr": current += "\n"297 case "w:noBreakHyphen": current += "-"298 case "w:tbl": tableDepth += 1299 case "w:tr": if tableDepth == 1 { rowCells = [] }300 case "w:tc": if tableDepth == 1 { cellParagraphs = [] }301 default: break302 }303 }304 func parser(_ parser: XMLParser, foundCharacters string: String) { if inText { current += string } }305 func parser(_ parser: XMLParser, didEndElement name: String, namespaceURI: String?, qualifiedName: String?) {306 switch name {307 case "w:t": inText = false308 case "w:p":309 if tableDepth > 0 { let t = current.trimmingCharacters(in: .whitespacesAndNewlines); if !t.isEmpty { cellParagraphs.append(t) } }310 else { blocks.append(Block(text: current, headingLevel: headingLevel, isRow: false)) }311 current = ""312 case "w:tc": if tableDepth == 1 { rowCells.append(cellParagraphs.joined(separator: " ")); cellParagraphs = [] }313 case "w:tr": if tableDepth == 1, rowCells.contains(where: { !$0.isEmpty }) { blocks.append(Block(text: rowCells.joined(separator: " | "), headingLevel: nil, isRow: true)) }314 case "w:tbl": tableDepth = max(0, tableDepth - 1)315 default: break316 }317 }318}319