SPB Git

Repository Public License

PipelineTests.swift 38.3 KB 660 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Scan9// File             : Packages/ORIXKit/Tests/ORIXKitTests/PipelineTests.swift10// Description      : Integration tests through the real engines: Vision11//                    recognition, searchable PDFs read back with PDFKit,12//                    storage, export, workflows.13// Module           : ORIXKitTests14// Repository       : https://git.spboucher.ai/orix-scan15// Created          : 2026-11-2016// Last Modified    : 2026-11-2017//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : Proprietary, all rights reserved (see license file)83// License File     : LICENSE84// Maintainer       : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import CoreGraphics92import Foundation93import PDFKit94import Testing95@testable import ORIXKit9697@Suite struct RecognitionTests {98    @Test func recognizesEnglish() async throws {99        let page = try await OCREngine().recognize(Fixture.page(Fixture.english), options: OCROptions(mode: .accurate, languages: ["en-US"]))100        let cer = TextMetrics.characterErrorRate(reference: Fixture.english, hypothesis: page.plainText)101        #expect(cer < 0.02, "CER \(cer): \(page.plainText)")102        #expect(page.lines.count == 5)103        #expect(page.lines.allSatisfy { !$0.words.isEmpty })104    }105106    @Test func recognizesFrenchAccents() async throws {107        let page = try await OCREngine().recognize(Fixture.page(Fixture.french), options: OCROptions(mode: .accurate, languages: ["fr-FR"]))108        let cer = TextMetrics.characterErrorRate(reference: Fixture.french, hypothesis: page.plainText)109        #expect(cer < 0.02, "CER \(cer): \(page.plainText)")110        #expect(page.plainText.contains("québécoise"))111    }112113    @Test func fastModeAlsoReads() async throws {114        let page = try await OCREngine().recognize(Fixture.page(Fixture.english), options: OCROptions(mode: .fast, languages: ["en-US"]))115        #expect(TextMetrics.characterErrorRate(reference: Fixture.english, hypothesis: page.plainText) < 0.08)116    }117118    @Test func supportedLanguagesComeFromVision() {119        let accurate = OCREngine.supportedLanguages(mode: .accurate)120        #expect(accurate.contains("en-US"))121        #expect(accurate.contains("fr-FR"))122        #expect(!OCREngine.supportedLanguages(mode: .fast).isEmpty)123    }124125    @Test func wordBoxesLieInsideTheirLine() async throws {126        let page = try await OCREngine().recognize(Fixture.page(Fixture.english))127        for line in page.lines {128            let outer = line.quad.bounds.insetBy(dx: -0.01, dy: -0.01)129            for word in line.words { #expect(outer.contains(word.quad.bounds), "\(word.text)") }130            #expect(line.words.map(\.text).joined(separator: " ") == line.text.split(whereSeparator: \.isWhitespace).joined(separator: " "))131        }132    }133134    @Test func measuresSkew() async throws {135        let page = try await OCREngine().recognize(Fixture.page(Fixture.english, rotationDegrees: 4), options: OCROptions(mode: .fast))136        let skew = try #require(page.skewDegrees)137        #expect(abs(skew - 4) < 0.7, "\(skew)")138    }139140    @Test func cancellationStopsRecognition() async throws {141        let task = Task { try await OCREngine().recognize(Fixture.page(Fixture.english, width: 3300, height: 4600, fontSize: 80)) }142        task.cancel()143        await #expect(throws: (any Error).self) { try await task.value }144    }145146    @Test func extractsTableFromRealRecognition() async throws {147        let rows = [["Item", "Quantity", "Price"], ["Paper", "12", "4.50"], ["Ink", "30", "19.99"], ["Stapler", "75", "7.25"], ["Folders", "25", "11.00"]]148        let image = Fixture.table(rows, columnX: [160, 760, 1240])149        let page = try await OCREngine().recognize(image, options: OCROptions(mode: .accurate, languages: ["en-US"]))150        let table = try #require(TableExtractor.extract(from: page).first)151        #expect(table.rows == rows, "\(table.rows)")152    }153}154155@Suite struct SearchablePDFTests {156    private func build(_ images: [CGImage], password: String? = nil, pageSize: PageSizeMode = .automatic) async throws -> (URL, [OCRPage]) {157        let url = Fixture.tempDirectory().appendingPathComponent("out.pdf")158        var pages: [OCRPage] = []159        for image in images { pages.append(try await OCREngine().recognize(image)) }160        let recognized = pages161        try await SearchablePDFBuilder.build(pageCount: images.count, to: url,162                                             options: PDFBuildOptions(pageSize: pageSize, title: "Rapport d'été", metadata: DocumentMetadata(author: "ORIX", subject: "Test", keywords: ["a", "b"]), password: password)) { i in163            .image(jpeg: Fixture.jpeg(images[i]), pixelWidth: images[i].width, pixelHeight: images[i].height, text: recognized[i])164        }165        return (url, pages)166    }167168    @Test func textIsExtractableAndSearchable() async throws {169        let (url, _) = try await build([Fixture.page(Fixture.english)])170        let document = try #require(PDFDocument(url: url))171        let text = try #require(document.page(at: 0)?.string)172        #expect(TextMetrics.characterErrorRate(reference: Fixture.english, hypothesis: text) < 0.03, "\(text)")173        #expect(document.findString("customer satisfaction", withOptions: [.caseInsensitive]).count == 1)174        #expect(document.findString("zebra", withOptions: [.caseInsensitive]).isEmpty)175    }176177    @Test func selectionSitsOnTheRecognizedWord() async throws {178        let (url, pages) = try await build([Fixture.page(Fixture.english)])179        let document = try #require(PDFDocument(url: url)), page = try #require(document.page(at: 0))180        let box = page.bounds(for: .mediaBox)181        for needle in ["Revenue", "Thursday", "quarterly"] {182            let selection = try #require(document.findString(needle, withOptions: []).first)183            let found = selection.bounds(for: page)184            let word = try #require(pages[0].lines.flatMap(\.words).first { $0.text.contains(needle) })185            let b = word.quad.bounds186            let expected = CGRect(x: b.minX * box.width, y: (1 - b.maxY) * box.height, width: b.width * box.width, height: b.height * box.height)187            let overlap = found.intersection(expected)188            #expect(!overlap.isNull && overlap.width * overlap.height > 0.5 * expected.width * expected.height, "\(needle): \(found) vs \(expected)")189            #expect(abs(found.midX - expected.midX) < expected.width * 0.25 + 2, "\(needle): \(found) vs \(expected)")190        }191    }192193    @Test func unicodeSurvives() async throws {194        let (url, _) = try await build([Fixture.page(Fixture.french)])195        let document = try #require(PDFDocument(url: url))196        #expect(document.findString("québécoise", withOptions: []).count == 1)197        #expect(document.findString("quebecoise", withOptions: [.diacriticInsensitive]).count == 1)198        #expect(document.findString("février", withOptions: []).count == 1)199        #expect(document.documentAttributes?[PDFDocumentAttribute.titleAttribute] as? String == "Rapport d'été")200        #expect(document.documentAttributes?[PDFDocumentAttribute.authorAttribute] as? String == "ORIX")201    }202203    @Test func multiPageOrderAndSizes() async throws {204        let a = Fixture.page("Alpha page one"), b = Fixture.page("Bravo page two", width: 2200, height: 1700), c = Fixture.page("Charlie page three", width: 900, height: 2400)205        let (url, _) = try await build([a, b, c])206        let document = try #require(PDFDocument(url: url))207        #expect(document.pageCount == 3)208        #expect(document.page(at: 0)?.string?.contains("Alpha") == true)209        #expect(document.page(at: 1)?.string?.contains("Bravo") == true)210        #expect(document.page(at: 2)?.string?.contains("Charlie") == true)211        let s0 = document.page(at: 0)!.bounds(for: .mediaBox).size, s1 = document.page(at: 1)!.bounds(for: .mediaBox).size, s2 = document.page(at: 2)!.bounds(for: .mediaBox).size212        #expect(abs(s0.width - 595.28) < 0.5 && abs(s0.height - 841.89) < 0.5)   // A4 portrait213        #expect(abs(s1.width - 792) < 0.5 && abs(s1.height - 612) < 0.5)         // Letter landscape214        #expect(abs(s2.height - 841.89) < 0.5 && abs(s2.width / s2.height - 900.0 / 2400.0) < 0.002) // receipt keeps its shape215    }216217    @Test func fixedPaperSizeKeepsTextAligned() async throws {218        let (url, _) = try await build([Fixture.page(Fixture.english, width: 1200, height: 2400)], pageSize: .letter)219        let document = try #require(PDFDocument(url: url))220        #expect(document.page(at: 0)!.bounds(for: .mediaBox).size == CGSize(width: 612, height: 792))221        #expect(document.findString("Operating costs", withOptions: []).count == 1)222    }223224    @Test func passwordProtects() async throws {225        let (url, _) = try await build([Fixture.page(Fixture.english)], password: "s3cret")226        let document = try #require(PDFDocument(url: url))227        #expect(document.isEncrypted && document.isLocked)228        #expect(!document.unlock(withPassword: "wrong"))229        #expect(document.unlock(withPassword: "s3cret"))230        #expect(document.findString("Revenue", withOptions: []).count == 1)231        #expect(throws: ORIXError.passwordRequired) { try PDFTools.open(url) }232        #expect(throws: ORIXError.wrongPassword) { try PDFTools.open(url, password: "nope") }233    }234235    @Test func withoutTextLayerNothingIsSearchable() async throws {236        let image = Fixture.page(Fixture.english)237        let url = Fixture.tempDirectory().appendingPathComponent("plain.pdf")238        try await SearchablePDFBuilder.build(pageCount: 1, to: url, options: PDFBuildOptions()) { _ in239            .image(jpeg: Fixture.jpeg(image), pixelWidth: image.width, pixelHeight: image.height, text: nil)240        }241        let plain: String = PDFDocument(url: url)?.page(at: 0)?.string ?? ""242        #expect(plain.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)243    }244245    @Test func emptyDocumentIsRefused() async {246        await #expect(throws: ORIXError.emptyDocument) {247            try await SearchablePDFBuilder.build(pageCount: 0, to: Fixture.tempDirectory().appendingPathComponent("x.pdf"), options: PDFBuildOptions()) { _ in248                .pdf(url: URL(fileURLWithPath: "/"), pageIndex: 0, quarterTurns: 0)249            }250        }251    }252}253254@Suite struct StoreTests {255    @Test func draftsAreInvisibleUntilCommitAndPurgedAfterACrash() async throws {256        let p = Fixture.processor()257        var draft = try await p.store.beginDraft(title: "Interrupted")258        try await p.store.addImage(Fixture.jpeg(Fixture.page("x")), fileExtension: "jpg", to: &draft)259        #expect(await p.store.allDocumentIDs().isEmpty)260        // Simulated termination: a new store over the same root finds the staged leftovers.261        let relaunched = try DocumentStore(root: p.store.root)262        #expect(await relaunched.recoverAfterLaunch() == 1)263        #expect(await relaunched.allDocumentIDs().isEmpty)264        #expect((try? FileManager.default.contentsOfDirectory(atPath: p.store.root.appendingPathComponent("Staging").path))?.isEmpty == true)265    }266267    @Test func committedDocumentSurvivesRelaunch() async throws {268        let p = Fixture.processor()269        let manifest = try await Fixture.document([Fixture.page("one"), Fixture.page("two")], title: "Reçu d'épicerie – été 2026 日本", in: p)270        let relaunched = try DocumentStore(root: p.store.root)271        await relaunched.recoverAfterLaunch()272        let loaded = try await relaunched.manifest(manifest.id)273        #expect(loaded.title == "Reçu d'épicerie – été 2026 日本")274        #expect(loaded.pages.count == 2)275        #expect(loaded.pages.allSatisfy { FileManager.default.fileExists(atPath: relaunched.fileURL(for: $0.source, in: loaded.id).path) })276        #expect(loaded.pages[0].width == 1654 && loaded.pages[0].height == 2339)277    }278279    @Test func emptyDraftAndBadImageAreRefused() async throws {280        let p = Fixture.processor()281        var draft = try await p.store.beginDraft(title: "Empty")282        await #expect(throws: ORIXError.unreadableImage) { try await p.store.addImage(Data("not an image".utf8), fileExtension: "jpg", to: &draft) }283        let empty = draft284        await #expect(throws: ORIXError.emptyDocument) { try await p.store.commit(empty) }285        #expect(await p.store.allDocumentIDs().isEmpty)286    }287288    @Test func corruptedManifestIsReportedNotCrashed() async throws {289        let p = Fixture.processor()290        let manifest = try await Fixture.document([Fixture.page("one")], in: p)291        try Data("{ broken".utf8).write(to: p.store.directory(for: manifest.id).appendingPathComponent("manifest.json"))292        await #expect(throws: ORIXError.corruptedManifest) { try await p.store.manifest(manifest.id) }293        await #expect(throws: ORIXError.documentNotFound) { try await p.store.manifest(UUID()) }294    }295296    @Test func pageOperations() async throws {297        let p = Fixture.processor()298        var manifest = try await Fixture.document([Fixture.page("one"), Fixture.page("two"), Fixture.page("three")], in: p)299        let first = manifest.pages[0]300        manifest = try await p.store.duplicatePage(first.id, in: manifest.id)301        #expect(manifest.pages.count == 4)302        #expect(manifest.pages[1].source == first.source && manifest.pages[1].id != first.id)303        // Deleting the original keeps the shared file because the duplicate still uses it.304        manifest = try await p.store.deletePages([first.id], in: manifest.id)305        #expect(manifest.pages.count == 3)306        #expect(FileManager.default.fileExists(atPath: p.store.fileURL(for: first.source, in: manifest.id).path))307        manifest = try await p.store.deletePages([manifest.pages[0].id], in: manifest.id)308        #expect(!FileManager.default.fileExists(atPath: p.store.fileURL(for: first.source, in: manifest.id).path))309        // Reorder and persist.310        manifest.pages.reverse()311        try await p.store.save(manifest)312        #expect(try await p.store.manifest(manifest.id).pages.map(\.id) == manifest.pages.map(\.id))313        let all = Set(manifest.pages.map(\.id)), id = manifest.id314        await #expect(throws: ORIXError.emptyDocument) { try await p.store.deletePages(all, in: id) }315    }316317    @Test func retakeReplacesTheCapture() async throws {318        let p = Fixture.processor()319        let manifest = try await Fixture.document([Fixture.page("old"), Fixture.page("keep")], in: p)320        let old = manifest.pages[0]321        let updated = try await p.store.replaceImage(page: old.id, in: manifest.id, with: Fixture.jpeg(Fixture.page("new", width: 1000, height: 1400)), fileExtension: "jpg")322        #expect(updated.pages.count == 2)323        #expect(updated.pages[0].width == 1000 && updated.pages[1].id == manifest.pages[1].id)324        #expect(!FileManager.default.fileExists(atPath: p.store.fileURL(for: old.source, in: manifest.id).path))325    }326327    @Test func mergeSplitAndExtract() async throws {328        let p = Fixture.processor()329        let a = try await Fixture.document([Fixture.page("a1"), Fixture.page("a2")], title: "A", in: p)330        let b = try await Fixture.document([Fixture.page("b1")], title: "B", in: p)331        let merged = try await p.store.merge([a.id, b.id], title: "A + B")332        #expect(merged.pages.count == 3)333        // Sources untouched, merged copy independent.334        try await p.store.deletePermanently(a.id)335        #expect(try await p.renderedImage(merged.pages[0], in: merged.id, maxDimension: 200).width > 0)336        let extracted = try await p.store.derive(from: merged.id, pages: [merged.pages[2].id, merged.pages[0].id], title: "Part")337        #expect(extracted.pages.count == 2)338        #expect(try await p.store.manifest(merged.id).pages.count == 3)339        let empty: [UUID] = [], mid = merged.id340        await #expect(throws: ORIXError.emptyDocument) { try await p.store.derive(from: mid, pages: empty, title: "None") }341    }342343    @Test func importsPDFsIncludingProtectedAndRefusesCorrupted() async throws {344        let p = Fixture.processor()345        let dir = Fixture.tempDirectory()346        let plain = dir.appendingPathComponent("Rapport d'été – 2026.pdf"), locked = dir.appendingPathComponent("locked.pdf"), bad = dir.appendingPathComponent("bad.pdf")347        Fixture.textPDF(pages: ["First vector page", "Second vector page"], at: plain)348        Fixture.textPDF(pages: ["Secret content"], size: CGSize(width: 300, height: 1200), password: "open-me", at: locked)349        try Data("%PDF-1.4 truncated garbage".utf8).write(to: bad)350351        var draft = try await p.store.beginDraft(title: "Imports")352        try await p.store.addPDF(at: plain, to: &draft)353        await #expect(throws: ORIXError.passwordRequired) { try await p.store.addPDF(at: locked, to: &draft) }354        await #expect(throws: ORIXError.wrongPassword) { try await p.store.addPDF(at: locked, password: "nope", to: &draft) }355        try await p.store.addPDF(at: locked, password: "open-me", to: &draft)356        await #expect(throws: ORIXError.unreadablePDF) { try await p.store.addPDF(at: bad, to: &draft) }357        let manifest = try await p.store.commit(draft)358        #expect(manifest.pages.count == 3 && manifest.kind == .imported)359        #expect(manifest.pages[2].width == 300 && manifest.pages[2].height == 1200)360        #expect(await p.text(of: manifest.pages[1], in: manifest.id) == "Second vector page")361        #expect(await p.text(of: manifest.pages[2], in: manifest.id) == "Secret content")362    }363364    @Test func lowStorageIsDetected() async throws {365        let p = Fixture.processor()366        #expect(throws: ORIXError.insufficientStorage(neededBytes: .max / 2)) { try p.store.ensureSpace(for: .max / 2) }367        try p.store.ensureSpace(for: 1024)368    }369}370371@Suite struct ProcessorTests {372    @Test func scanToSearchablePDFEndToEnd() async throws {373        let p = Fixture.processor()374        let manifest = try await Fixture.document([Fixture.page(Fixture.english), Fixture.page(Fixture.french)], title: "Rapport", in: p)375        let summary = try await p.recognize(manifest.id, options: OCROptions())376        #expect(summary.pagesRecognized == 2 && summary.lineCount == 10)377        let again = try await p.recognize(manifest.id, options: OCROptions())378        #expect(again.pagesRecognized == 0 && again.pagesSkipped == 2)379        let export = try await p.exportPDF(manifest.id, options: ExportOptions(preset: .balanced))380        #expect(export.url.lastPathComponent == "Rapport.pdf")381        #expect(export.pageCount == 2 && export.pagesWithTextLayer == 2 && export.byteSize > 0)382        #expect(PDFTools.matchCount(of: "customer satisfaction", in: export.url) == 1)383        #expect(PDFTools.matchCount(of: "clientèle québécoise", in: export.url) == 1)384        let text = try await p.fullText(of: manifest.id)385        #expect(TextMetrics.characterErrorRate(reference: Fixture.english + "\n" + Fixture.french, hypothesis: text) < 0.02)386    }387388    @Test func changingGeometryInvalidatesText() async throws {389        let p = Fixture.processor()390        let manifest = try await Fixture.document([Fixture.page(Fixture.english)], in: p)391        try await p.recognize(manifest.id, options: OCROptions(mode: .fast))392        let page = manifest.pages[0]393        // A colour-only change keeps the text…394        try await p.updateEdits(in: manifest.id) { edit, _ in edit.filter = .grayscale; edit.contrast = 0.3 }395        #expect(await p.store.ocr(page: page.id, in: manifest.id) != nil)396        // …a rotation discards it, so an outdated text layer cannot be exported.397        let rotated = try await p.updateEdits(in: manifest.id) { edit, _ in edit.quarterTurns = 1 }398        #expect(await p.store.ocr(page: page.id, in: manifest.id) == nil)399        let image = try await p.renderedImage(rotated.pages[0], in: manifest.id)400        #expect(image.width == 2339 && image.height == 1654)401        let export = try await p.exportPDF(manifest.id)402        #expect(export.pagesWithTextLayer == 0)403    }404405    @Test func rotatedPageIsRecognizedAfterRotation() async throws {406        let p = Fixture.processor()407        // A page captured sideways: rotate it back with a quarter turn, then recognize.408        let upright = Fixture.page(Fixture.english)409        let ctx = CGContext(data: nil, width: upright.height, height: upright.width, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpace(name: CGColorSpace.sRGB)!,410                            bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!411        ctx.translateBy(x: 0, y: CGFloat(upright.width)); ctx.rotate(by: -.pi / 2)412        ctx.draw(upright, in: CGRect(x: 0, y: 0, width: upright.width, height: upright.height))413        let manifest = try await Fixture.document([ctx.makeImage()!], in: p)414        var best = 1.0415        for turns in [1, 3] {416            try await p.updateEdits(in: manifest.id) { edit, _ in edit.quarterTurns = turns }417            try await p.recognize(manifest.id, options: OCROptions())418            best = min(best, TextMetrics.characterErrorRate(reference: Fixture.english, hypothesis: try await p.fullText(of: manifest.id)))419        }420        #expect(best < 0.02)421    }422423    @Test func deskewStraightensATiltedPage() async throws {424        let p = Fixture.processor()425        let manifest = try await Fixture.document([Fixture.page(Fixture.english, rotationDegrees: 5)], in: p)426        #expect(try await p.deskew(manifest.id) == 1)427        let updated = try await p.store.manifest(manifest.id)428        #expect(abs(updated.pages[0].edit.deskewDegrees + 5) < 0.8, "\(updated.pages[0].edit.deskewDegrees)")429        let image = try await p.renderedImage(updated.pages[0], in: manifest.id, maxDimension: 1600)430        let residual = try await OCREngine().recognize(image, options: OCROptions(mode: .fast)).skewDegrees ?? 0431        #expect(abs(residual) < 0.8, "\(residual)")432        #expect(try await p.deskew(manifest.id) == 0)433    }434435    @Test func compressionPresetsReportRealSizesAndStayReadable() async throws {436        let p = Fixture.processor()437        let manifest = try await Fixture.document([Fixture.page(Fixture.english, width: 2480, height: 3508, fontSize: 60, margin: 200)], in: p, edit: .untouched)438        try await p.recognize(manifest.id, options: OCROptions())439        var sizes: [ExportPreset: Int64] = [:]440        for preset in ExportPreset.allCases {441            let export = try await p.exportPDF(manifest.id, options: ExportOptions(preset: preset))442            sizes[preset] = export.byteSize443            #expect(export.byteSize == Int64(try export.url.resourceValues(forKeys: [.fileSizeKey]).fileSize!))444            // Readability after compression: rasterize the exported page and recognize it again.445            let cg = try #require(CGPDFDocument(export.url as CFURL)?.page(at: 1))446            let raster = try p.renderer.rasterize(pdfPage: cg, dpi: 200)447            let reread = try await OCREngine().recognize(raster, options: OCROptions(mode: .accurate, languages: ["en-US"]))448            #expect(TextMetrics.characterErrorRate(reference: Fixture.english, hypothesis: reread.plainText) < 0.03, "\(preset)")449            #expect(PDFTools.matchCount(of: "Revenue increased", in: export.url) == 1)450        }451        #expect(sizes[.small]! < sizes[.balanced]! && sizes[.balanced]! < sizes[.maximum]!, "\(sizes)")452    }453454    @Test func importedTextPagesPassThroughUntouched() async throws {455        let p = Fixture.processor()456        let pdf = Fixture.tempDirectory().appendingPathComponent("vector.pdf")457        Fixture.textPDF(pages: ["Vector text stays vector"], at: pdf)458        var draft = try await p.store.beginDraft(title: "Mixed")459        try await p.store.addPDF(at: pdf, to: &draft)460        try await p.store.addImage(Fixture.jpeg(Fixture.page(Fixture.english)), fileExtension: "jpg", to: &draft)461        let manifest = try await p.store.commit(draft)462        #expect(manifest.kind == .mixed)463        let summary = try await p.recognize(manifest.id, options: OCROptions())464        #expect(summary.pagesRecognized == 1 && summary.pagesSkipped == 1)465        let export = try await p.exportPDF(manifest.id, options: ExportOptions(preset: .small))466        #expect(export.pagesPassedThrough == 1 && export.pagesRecompressed == 1)467        #expect(PDFTools.matchCount(of: "Vector text stays vector", in: export.url) == 1)468        #expect(PDFTools.matchCount(of: "lazy dog", in: export.url) == 1)469        // Rotating an imported page is applied on export.470        try await p.updateEdits(in: manifest.id, pages: [manifest.pages[0].id]) { edit, _ in edit.quarterTurns = 1 }471        let rotated = try await p.exportPDF(manifest.id, options: ExportOptions(preset: .maximum))472        #expect(PDFDocument(url: rotated.url)?.page(at: 0)?.rotation == 90)473    }474475    @Test func imageOnlyPDFPagesAreRecognizedAndCompressed() async throws {476        let p = Fixture.processor()477        // A "scanned PDF": pages are images without text.478        let source = Fixture.tempDirectory().appendingPathComponent("scan.pdf")479        let image = Fixture.page(Fixture.english, width: 2480, height: 3508, fontSize: 60, margin: 200)480        try await SearchablePDFBuilder.build(pageCount: 1, to: source, options: PDFBuildOptions()) { _ in481            .image(jpeg: Fixture.jpeg(image, quality: 0.95), pixelWidth: image.width, pixelHeight: image.height, text: nil)482        }483        var draft = try await p.store.beginDraft(title: "Scanned PDF")484        try await p.store.addPDF(at: source, to: &draft)485        let manifest = try await p.store.commit(draft)486        let summary = try await p.recognize(manifest.id, options: OCROptions())487        #expect(summary.pagesRecognized == 1)488        let kept = try await p.exportPDF(manifest.id, options: ExportOptions(preset: .maximum))489        let small = try await p.exportPDF(manifest.id, options: ExportOptions(preset: .small))490        #expect(kept.pagesPassedThrough == 1 && small.pagesRecompressed == 1)491        #expect(small.byteSize < kept.byteSize, "\(small.byteSize) vs \(kept.byteSize)")492        #expect(PDFTools.matchCount(of: "customer satisfaction", in: small.url) == 1)493    }494495    @Test func textAndTableExports() async throws {496        let p = Fixture.processor()497        let rows = [["Item", "Quantity", "Price"], ["Paper", "12", "4.50"], ["Ink", "30", "19.99"], ["Stapler", "75", "7.25"]]498        let manifest = try await Fixture.document([Fixture.table(rows, columnX: [160, 760, 1240])], title: "Facture: été/2026", in: p, edit: .untouched)499        #expect(try await p.exportTables(manifest.id).isEmpty)500        try await p.recognize(manifest.id, options: OCROptions(mode: .accurate, languages: ["en-US"]))501        let csv = try #require(try await p.exportTables(manifest.id).first)502        #expect(csv.lastPathComponent == "Facture  été 2026.csv")503        let content = try String(contentsOf: csv, encoding: .utf8)504        #expect(content.contains("Paper,12,4.50\r\n"))505        let txt = try await p.exportText(manifest.id)506        #expect(try String(contentsOf: txt, encoding: .utf8).contains("Stapler"))507        #expect(try await p.exportImages(manifest.id).count == 1)508    }509510    @Test func filtersProduceDistinctValidImages() async throws {511        let p = Fixture.processor()512        // Uneven lighting: a grey gradient "shadow" over the page.513        let base = Fixture.page(Fixture.english, background: 0.72, ink: 0.1)514        let manifest = try await Fixture.document([base], in: p)515        var means: [FilterMode: Double] = [:]516        for mode in FilterMode.allCases {517            var edit = PageEdit(filter: mode)518            edit.shadowReduction = true519            let image = try await p.renderedImage(manifest.pages[0], in: manifest.id, edit: edit, maxDimension: 1200)520            #expect(max(image.width, image.height) == 1200)521            means[mode] = GreyBuffer(image, longestSide: 400).percentiles().2522        }523        // Enhancement lifts the grey paper toward white; the original stays as captured.524        #expect(abs(means[.original]! - 0.72) < 0.05, "\(means)")525        #expect(means[.color]! > 0.9 && means[.grayscale]! > 0.9 && means[.blackWhite]! > 0.93, "\(means)")526    }527528    @Test func blackAndWhiteKeepsFaintText() async throws {529        let p = Fixture.processor()530        // Faint pencil-like strokes: ink only 25 % darker than the paper.531        let manifest = try await Fixture.document([Fixture.page(Fixture.english, background: 0.92, ink: 0.68)], in: p, edit: PageEdit(filter: .blackWhite))532        let image = try await p.renderedImage(manifest.pages[0], in: manifest.id)533        let read = try await OCREngine().recognize(image, options: OCROptions(mode: .accurate, languages: ["en-US"]))534        #expect(TextMetrics.characterErrorRate(reference: Fixture.english, hypothesis: read.plainText) < 0.03, "\(read.plainText)")535    }536537    @Test func perspectiveCropRendersTheChosenRegion() async throws {538        let p = Fixture.processor()539        let manifest = try await Fixture.document([Fixture.page(Fixture.english)], in: p)540        let quad = Quad(rect: CGRect(x: 0, y: 0, width: 1, height: 0.5))541        let image = try await p.renderedImage(manifest.pages[0], in: manifest.id, edit: PageEdit(quad: quad, filter: .original, sharpen: 0))542        #expect(abs(image.width - 1654) <= 2 && abs(image.height - 1170) <= 2, "\(image.width)x\(image.height)")543        let read = try await OCREngine().recognize(image)544        #expect(read.plainText.contains("quarterly report"))545    }546547    @Test func previewCacheIsKeyedByEdit() async throws {548        let p = Fixture.processor()549        var manifest = try await Fixture.document([Fixture.page("cache")], in: p)550        let first = try p.cachedPreviewURL(manifest.pages[0], in: manifest.id, maxDimension: 400)551        #expect(try p.cachedPreviewURL(manifest.pages[0], in: manifest.id, maxDimension: 400) == first)552        manifest = try await p.updateEdits(in: manifest.id) { edit, _ in edit.filter = .blackWhite }553        #expect(try p.cachedPreviewURL(manifest.pages[0], in: manifest.id, maxDimension: 400) != first)554        await p.store.clearAllCaches()555        #expect(!FileManager.default.fileExists(atPath: first.path))556    }557558    @Test func exportCanBeCancelled() async throws {559        let p = Fixture.processor()560        let manifest = try await Fixture.document((0..<6).map { Fixture.page("Page \($0)") }, in: p)561        let task = Task { try await p.exportPDF(manifest.id) }562        task.cancel()563        await #expect(throws: ORIXError.cancelled) { try await task.value }564        #expect((try? FileManager.default.contentsOfDirectory(atPath: p.store.exportsDirectory.path))?.isEmpty == true)565    }566}567568@Suite struct WorkflowEngineTests {569    @Test func scanToSearchablePDFPreset() async throws {570        let p = Fixture.processor()571        let manifest = try await Fixture.document([Fixture.page(Fixture.english)], title: "Scan", in: p, edit: .untouched)572        let workflow = try #require(Workflow.presets.first { $0.presetKey == "scanToSearchablePDF" })573        let seen = Progress()574        let result = try await WorkflowEngine(processor: p).run(workflow, on: manifest.id) { seen.record($0.overall) }575        #expect(result.succeeded && result.outcomes.count == 3)576        let pdf = try #require(result.artifacts.first)577        #expect(PDFTools.matchCount(of: "lazy dog", in: pdf) == 1)578        #expect(try await p.store.manifest(manifest.id).pages[0].edit.filter == .color)579        let values = seen.values580        #expect(values.first == 0 && values.last == 1 && values == values.sorted())581    }582583    @Test func receiptArchiveRenamesAndFiles() async throws {584        let p = Fixture.processor()585        let manifest = try await Fixture.document([Fixture.page("Total 42.00")], title: "Scan", in: p)586        let workflow = try #require(Workflow.presets.first { $0.presetKey == "receiptArchive" })587        let result = try await WorkflowEngine(processor: p).run(workflow, on: manifest.id, now: Date(timeIntervalSince1970: 1_795_000_000))588        #expect(result.succeeded)589        #expect(result.folderName == "Receipts")590        #expect(result.newTitle?.hasPrefix("Receipt 2026-11-18") == true)591        #expect(try await p.store.manifest(manifest.id).title == result.newTitle)592        #expect(result.artifacts.isEmpty)593    }594595    @Test func extractTablesReportsWhenThereIsNone() async throws {596        let p = Fixture.processor()597        let manifest = try await Fixture.document([Fixture.page(Fixture.english)], in: p)598        let workflow = try #require(Workflow.presets.first { $0.presetKey == "extractTables" })599        let result = try await WorkflowEngine(processor: p).run(workflow, on: manifest.id)600        #expect(result.succeeded && result.artifacts.isEmpty)601        #expect(result.outcomes.last?.status == .nothingToDo(.noTablesFound))602    }603604    @Test func everyPresetRunsOnARealDocument() async throws {605        let p = Fixture.processor()606        for preset in Workflow.presets {607            let manifest = try await Fixture.document([Fixture.page(Fixture.french)], title: preset.name, in: p)608            let result = try await WorkflowEngine(processor: p, languages: ["fr-FR", "en-US"]).run(preset, on: manifest.id)609            #expect(result.succeeded, "\(preset.name): \(result.outcomes)")610            #expect(result.outcomes.count == preset.steps.count)611            for url in result.artifacts { #expect(FileManager.default.fileExists(atPath: url.path)) }612        }613    }614615    @Test func failingStepStopsTheRunAndKeepsTheDocument() async throws {616        let p = Fixture.processor()617        let manifest = try await Fixture.document([Fixture.page("x")], in: p)618        // Break the page file so rendering fails in the export step.619        try FileManager.default.removeItem(at: p.store.fileURL(for: manifest.pages[0].source, in: manifest.id))620        let workflow = Workflow(name: "Broken", steps: [ActionStep(.rename, pattern: "Renamed"), ActionStep(.exportPDF, searchable: false), ActionStep(.markFavorite)])621        let result = try await WorkflowEngine(processor: p).run(workflow, on: manifest.id)622        #expect(!result.succeeded)623        #expect(result.outcomes.map(\.status).last == .skipped)624        #expect(!result.markFavorite)625        #expect(try await p.store.manifest(manifest.id).title == "Renamed")626    }627628    @Test func cancellation() async throws {629        let p = Fixture.processor()630        let manifest = try await Fixture.document((0..<4).map { _ in Fixture.page(Fixture.english) }, in: p)631        let workflow = try #require(Workflow.presets.first { $0.presetKey == "bookScanner" })632        let task = Task { try await WorkflowEngine(processor: p).run(workflow, on: manifest.id) }633        task.cancel()634        await #expect(throws: ORIXError.cancelled) { _ = try await task.value }635        #expect(try await p.store.manifest(manifest.id).pages.count == 4)636    }637}638639final class Progress: @unchecked Sendable {640    private let lock = NSLock()641    private var storage: [Double] = []642    func record(_ v: Double) { lock.lock(); storage.append(v); lock.unlock() }643    var values: [Double] { lock.lock(); defer { lock.unlock() }; return storage }644}645646@Suite struct EnhancementQualityTests {647    /// Background cleanup must flatten the paper without leaving a grey halo around text.648    @Test func noHaloAroundText() async throws {649        let p = Fixture.processor()650        let manifest = try await Fixture.document([Fixture.page(Fixture.english + "\n" + Fixture.french, background: 0.8, ink: 0.1)], in: p)651        let image = try await p.renderedImage(manifest.pages[0], in: manifest.id, edit: PageEdit(filter: .color, sharpen: 0), maxDimension: 1200)652        let grey = GreyBuffer(image, longestSide: 1200)653        var histogram = [Int](repeating: 0, count: 256)654        for v in grey.pixels { histogram[Int(v)] += 1 }655        // Pixels that are neither ink (< 0.5) nor clean paper (>= 0.93): halo and anti-aliased stroke edges together.656        let murky = histogram[128..<237].reduce(0, +)657        #expect(Double(murky) / Double(grey.pixels.count) < 0.03, "\(Double(murky) / Double(grey.pixels.count))")658    }659}660
Files main
Menu

Theme