SPB Git

Repository Public

PDFTextImporter.swift 9.5 KB 153 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/PDFTextImporter.swift3// Description: PDF import limited to the text layer (PDFKit): paragraphs are rebuilt heuristically page by page and page breaks4//              are kept. Layout, fonts, images and tables are not reconstructed, which the result always says.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import PDFKit10import DocModel11import DocContainer1213public enum PDFTextImporter {14    /// Never throws: unreadable, locked or text-less PDFs yield an empty document and a `.dropped` warning. Use `read` to get the error instead.15    public static func importDocument(_ data: Data) -> ImportResult {16        do { return try read(data) } catch { return ImportFailure.result(error, format: .pdf) }17    }1819    /// Throws `FormatError.unreadable` for data that is not a PDF, for a password-protected PDF and for a PDF without any text layer (a scan).20    public static func read(_ data: Data) throws -> ImportResult {21        guard data.count <= FormatLimits.maxPDFBytes else { throw FormatError.tooLarge("the PDF file is larger than \(FormatLimits.maxPDFBytes) bytes") }22        guard data.prefix(1024).range(of: Data("%PDF-".utf8)) != nil, let pdf = PDFDocument(data: data) else {23            throw FormatError.unreadable("the file is not a PDF document")24        }25        if pdf.isLocked, !pdf.unlock(withPassword: "") {26            throw FormatError.unreadable("the PDF is password-protected; remove the password before importing it")27        }28        guard pdf.pageCount > 0 else { throw FormatError.unreadable("the PDF has no pages") }29        guard pdf.pageCount <= FormatLimits.maxPDFPages else { throw FormatError.tooLarge("the PDF has more than \(FormatLimits.maxPDFPages) pages") }30        var warnings = WarningCollector()31        if !pdf.allowsCopying {32            throw FormatError.unreadable("the PDF does not allow text extraction (copying is restricted by its owner)")33        }34        // Lines with their position, page by page. The vertical gaps tell paragraphs apart far better than the text alone.35        var pages: [[Line]] = []36        var pitches: [Double] = [], heights: [Double] = []37        for index in 0..<pdf.pageCount {38            guard let page = pdf.page(at: index) else { pages.append([]); continue }39            var lines: [Line] = []40            if let selection = page.selection(for: page.bounds(for: .mediaBox)) {41                for line in selection.selectionsByLine() {42                    let text = TextSanitizer.clean(line.string ?? "").replacingOccurrences(of: lineSeparatorCharacter, with: " ").trimmingCharacters(in: .whitespaces)43                    guard !text.isEmpty else { continue }44                    let box = line.bounds(for: page)45                    lines.append(Line(text: text, x: Double(box.minX), y: Double(box.midY), height: Double(box.height)))46                }47            }48            if lines.isEmpty {49                // No line geometry: fall back to the page text, where only blank lines separate paragraphs for sure.50                lines = PDFTextImporter.lines(fromText: page.string ?? "")51            }52            for (previous, current) in zip(lines, lines.dropFirst()) {53                if let a = previous.y, let b = current.y, a - b > 0.5 { pitches.append(a - b) }54            }55            heights.append(contentsOf: lines.compactMap(\.height).filter { $0 > 0 })56            pages.append(lines)57        }58        func median(_ values: [Double]) -> Double? { values.isEmpty ? nil : values.sorted()[values.count / 2] }59        // The usual distance between two lines of running text; with too few samples, a line height and a half.60        let typicalPitch = pitches.count >= 3 ? median(pitches) : median(heights).map { $0 * 1.3 }61        var body: [Block] = []62        var emptyPages = 0, joinedHyphens = 063        for (index, lines) in pages.enumerated() {64            let paragraphs = rebuildParagraphs(lines, typicalPitch: typicalPitch, joinedHyphens: &joinedHyphens)65            if paragraphs.isEmpty { emptyPages += 1 }66            if index > 0 { body.append(.pageBreak(BreakBlock())) }67            body.append(contentsOf: paragraphs.map { .paragraph(Paragraph($0)) })68        }69        guard emptyPages < pdf.pageCount else {70            throw FormatError.unreadable("the PDF has no text layer (it is probably a scan); run text recognition (OCR) on it first")71        }72        warnings.add(.approximated, "PDF layout", "Only the text of the PDF was imported. Paragraphs were rebuilt heuristically; layout, fonts, images, tables, headers and footers are not reconstructed.")73        if emptyPages > 0 {74            warnings.add(.dropped, "Pages without text", "\(emptyPages) page\(emptyPages == 1 ? " has" : "s have") no text layer (scanned or image-only) and \(emptyPages == 1 ? "is" : "are") empty in the document.")75        }76        if joinedHyphens > 0 {77            warnings.add(.approximated, "Hyphenation", "\(joinedHyphens) word\(joinedHyphens == 1 ? "" : "s") split by a hyphen at the end of a line \(joinedHyphens == 1 ? "was" : "were") joined; check compound words.")78        }79        var meta = DocumentMetadata()80        if let attributes = pdf.documentAttributes {81            if let t = attributes[PDFDocumentAttribute.titleAttribute] as? String { meta.title = TextSanitizer.clean(t) }82            if let a = attributes[PDFDocumentAttribute.authorAttribute] as? String { meta.author = TextSanitizer.clean(a) }83            if let s = attributes[PDFDocumentAttribute.subjectAttribute] as? String { meta.subject = TextSanitizer.clean(s) }84            if let k = attributes[PDFDocumentAttribute.keywordsAttribute] as? [String] { meta.keywords = k.map(TextSanitizer.clean) }85        }86        var setup = PageSetup()87        if let bounds = pdf.page(at: 0)?.bounds(for: .mediaBox), bounds.width >= 72, bounds.height >= 72 {88            let w = Double(min(bounds.width, bounds.height)), h = Double(max(bounds.width, bounds.height))89            setup.paper = PaperSize.all.first { abs($0.width - w) < 1.5 && abs($0.height - h) < 1.5 } ?? PaperSize(name: "Custom", width: w, height: h)90            setup.landscape = bounds.width > bounds.height91        }92        return ImportResult(document: DocumentModel(meta: meta, pageSetup: setup, body: body), warnings: warnings.warnings)93    }9495    /// One line of extracted text. `x` / `y` / `height` are page coordinates when the PDF gave them; a nil `text` marks a blank line.96    struct Line {97        var text: String?98        var x: Double?99        var y: Double?100        var height: Double?101    }102103    static func lines(fromText pageText: String) -> [Line] {104        pageText.split(omittingEmptySubsequences: false, whereSeparator: { $0 == "\n" || $0 == "\r" || $0 == "\u{2028}" || $0 == "\u{2029}" || $0 == "\u{0C}" })105            .map { TextSanitizer.clean(String($0)).trimmingCharacters(in: .whitespaces) }106            .map { Line(text: $0.isEmpty ? nil : $0) }107    }108109    /// Text-only variant: a blank line always separates paragraphs; otherwise a line that ends a sentence and is clearly shorter110    /// than the page's long lines ends its paragraph. Words hyphenated across lines are joined.111    static func rebuildParagraphs(_ pageText: String, joinedHyphens: inout Int) -> [String] {112        rebuildParagraphs(lines(fromText: pageText), typicalPitch: nil, joinedHyphens: &joinedHyphens)113    }114115    /// Lines → paragraphs. A new paragraph starts after a blank line, after a vertical gap clearly larger than the usual line pitch,116    /// when the text jumps back up (next column), on a first-line indent, and after a short line that ends a sentence.117    static func rebuildParagraphs(_ lines: [Line], typicalPitch: Double?, joinedHyphens: inout Int) -> [String] {118        let lengths = lines.compactMap { $0.text?.count }.sorted()119        guard !lengths.isEmpty else { return [] }120        // The 90th percentile stands for a "full" line.121        let full = lengths[min(lengths.count - 1, lengths.count * 9 / 10)]122        let left = lines.compactMap(\.x).min()123        var out: [String] = []124        var current = ""125        var previous: Line?126        var previousEndsParagraph = false127        func flush() { if !current.isEmpty { out.append(current); current = "" } }128        for line in lines {129            guard let text = line.text else { flush(); previous = nil; continue }130            var startsNew = previousEndsParagraph131            if let p = previous, let py = p.y, let y = line.y {132                let gap = py - y133                if gap < -1 { startsNew = true }                                            // back up the page: another column or block134                else if let pitch = typicalPitch, gap > pitch * 1.45 { startsNew = true }135            }136            if let p = previous, let px = p.x, let x = line.x, let left, px <= left + 2, x > left + 9 { startsNew = true }      // first-line indent137            if startsNew { flush() }138            if current.isEmpty { current = text }139            else if current.hasSuffix("-"), current.count >= 2, current.dropLast().last?.isLetter == true, text.first?.isLowercase == true {140                current.removeLast()141                current += text142                joinedHyphens += 1143            } else { current += " " + text }144            let endsSentence = text.last.map { ".!?:;\u{2026}\u{201D}\"»".contains($0) } ?? false145            let short = Double(text.count) < Double(full) * 0.72146            previousEndsParagraph = short && (endsSentence || (line.y == nil && (full < 20 || Double(text.count) < Double(full) * 0.45)))147            previous = line148        }149        flush()150        return out151    }152}153
Files main
Menu

Theme