SPB Git

Repository Public License

BenchmarkTests.swift 12.7 KB 207 lines swift

1// ORIX Scan2// File: Packages/ORIXKit/Tests/ORIXKitTests/BenchmarkTests.swift3// Description: OCR accuracy benchmark (CER / WER on a synthetic corpus with known ground truth) and engine timings.4//              Runs only when ORIX_BENCH is set; writes JSON to ORIX_BENCH_OUT. See Scripts/benchmark.sh.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import CoreGraphics9import CoreImage10import Foundation11import Testing12@testable import ORIXKit1314private let benchEnabled = ProcessInfo.processInfo.environment["ORIX_BENCH"] != nil1516struct CorpusCase {17    var name: String18    var category: String19    var truth: String20    var image: CGImage21}2223enum Corpus {24    static let mixed = """25    Meeting notes / Notes de réunion26    The budget was approved on Tuesday after a long discussion.27    Le budget a été approuvé mardi après une longue discussion.28    Next steps: hire two engineers and open the Québec office.29    Prochaines étapes : embaucher deux ingénieurs à Montréal.30    """3132    static let receipt = """33    CAFE DES ARTISANS34    87 rue Saint-Jean35    Latte grand 5.7536    Croissant beurre 3.9537    Sandwich jambon 9.2538    Sous-total 18.9539    Taxes 2.8440    TOTAL 21.7941    """4243    static let dense = """44    Section 4.2 describes the retention policy that applies to scanned records.45    Documents are stored locally and are never transmitted to a remote server.46    Each page keeps its original capture, so every edit can be reverted later.47    Recognition quality depends on resolution, contrast, focus and lighting.48    Tables are rebuilt from the position of words and may need a manual check.49    Exported files embed the image and an invisible, selectable text layer.50    """5152    static func lighting(_ image: CGImage) -> CGImage {53        // A strong diagonal shadow, as from a hand or a lamp at the side.54        let ctx = CGContext(data: nil, width: image.width, height: image.height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpace(name: CGColorSpace.sRGB)!,55                            bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!56        ctx.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))57        let colors = [CGColor(red: 0, green: 0, blue: 0, alpha: 0), CGColor(red: 0, green: 0, blue: 0, alpha: 0.62)] as CFArray58        let gradient = CGGradient(colorsSpace: CGColorSpace(name: CGColorSpace.sRGB)!, colors: colors, locations: [0, 1])!59        ctx.drawLinearGradient(gradient, start: CGPoint(x: 0, y: CGFloat(image.height)), end: CGPoint(x: CGFloat(image.width), y: 0), options: [])60        return ctx.makeImage()!61    }6263    static func twoColumns() -> (CGImage, String) {64        let left = ["The left column comes first", "and continues on this line", "before the reader moves on."]65        let right = ["The right column follows", "with its own three lines", "that end the page."]66        let rows = (0..<3).map { [left[$0], right[$0]] }67        return (Fixture.table(rows, columnX: [120, 900], width: 1654, height: 700, fontSize: 34), (left + right).joined(separator: "\n"))68    }6970    static func all() -> [CorpusCase] {71        let (columns, columnsTruth) = twoColumns()72        return [73            CorpusCase(name: "English, clean print", category: "English", truth: Fixture.english, image: Fixture.page(Fixture.english)),74            CorpusCase(name: "English, dense 10 pt-equivalent", category: "English", truth: dense, image: Fixture.page(dense, fontSize: 28)),75            CorpusCase(name: "English, serif (Times)", category: "English", truth: Fixture.english, image: Fixture.page(Fixture.english, fontName: "Times New Roman")),76            CorpusCase(name: "French, clean print", category: "French", truth: Fixture.french, image: Fixture.page(Fixture.french)),77            CorpusCase(name: "French, serif (Georgia)", category: "French", truth: Fixture.french, image: Fixture.page(Fixture.french, fontName: "Georgia")),78            CorpusCase(name: "Mixed English / French", category: "Mixed", truth: mixed, image: Fixture.page(mixed)),79            CorpusCase(name: "Receipt, monospaced, narrow", category: "Receipt", truth: receipt,80                       image: Fixture.page(receipt, width: 800, height: 1500, fontSize: 34, fontName: "Courier", margin: 70)),81            CorpusCase(name: "Low resolution (75 dpi-equivalent)", category: "Low resolution", truth: Fixture.english,82                       image: Fixture.page(Fixture.english, width: 620, height: 877, fontSize: 15, margin: 52)),83            CorpusCase(name: "Skewed 3°", category: "Skew", truth: Fixture.english, image: Fixture.page(Fixture.english, rotationDegrees: 3)),84            CorpusCase(name: "Skewed 8°", category: "Skew", truth: Fixture.english, image: Fixture.page(Fixture.english, rotationDegrees: 8)),85            CorpusCase(name: "Two columns", category: "Multi-column", truth: columnsTruth, image: columns),86            CorpusCase(name: "Strong side shadow", category: "Difficult lighting", truth: Fixture.french, image: lighting(Fixture.page(Fixture.french, background: 0.9))),87            CorpusCase(name: "Dark page", category: "Difficult lighting", truth: Fixture.english, image: Fixture.page(Fixture.english, background: 0.38, ink: 0.04)),88            CorpusCase(name: "Faint ink", category: "Difficult lighting", truth: Fixture.english, image: Fixture.page(Fixture.english, background: 0.9, ink: 0.66)),89            CorpusCase(name: "Out of focus (blur σ 2.5)", category: "Blur", truth: Fixture.english, image: Fixture.blurred(Fixture.page(Fixture.english), radius: 2.5)),90            CorpusCase(name: "Script font (Snell Roundhand)", category: "Handwriting-like", truth: Fixture.english, image: Fixture.page(Fixture.english, fontSize: 48, fontName: "Snell Roundhand")),91            CorpusCase(name: "Casual script (Bradley Hand)", category: "Handwriting-like", truth: Fixture.french, image: Fixture.page(Fixture.french, fontSize: 44, fontName: "Bradley Hand")),92        ]93    }94}9596@Suite(.enabled(if: benchEnabled), .serialized) struct Benchmarks {97    private func write(_ name: String, _ object: Any) throws {98        guard let out = ProcessInfo.processInfo.environment["ORIX_BENCH_OUT"] else { return }99        try FileManager.default.createDirectory(atPath: out, withIntermediateDirectories: true)100        let data = try JSONSerialization.data(withJSONObject: object, options: [.prettyPrinted, .sortedKeys])101        try data.write(to: URL(fileURLWithPath: out).appendingPathComponent(name))102    }103104    private func median(_ values: [Double]) -> Double { values.sorted()[values.count / 2] }105106    private func residentMegabytes() -> Double {107        var info = task_vm_info_data_t()108        var count = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)109        let result = withUnsafeMutablePointer(to: &info) { $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) } }110        return result == KERN_SUCCESS ? Double(info.phys_footprint) / 1_048_576 : 0111    }112113    @Test func ocrAccuracy() async throws {114        var rows: [[String: Any]] = []115        let engine = OCREngine()116        for item in Corpus.all() {117            var row: [String: Any] = ["name": item.name, "category": item.category, "characters": TextMetrics.normalize(item.truth).count,118                                      "pixels": "\(item.image.width)x\(item.image.height)"]119            for mode in [RecognitionMode.automatic, .accurate, .fast] {120                let options = OCROptions(mode: mode, languages: mode == .automatic ? [] : ["en-US", "fr-FR"])121                let page = try await engine.recognize(item.image, options: options)122                row["cer_\(mode.rawValue)"] = (TextMetrics.characterErrorRate(reference: item.truth, hypothesis: page.plainText) * 10000).rounded() / 10000123                row["wer_\(mode.rawValue)"] = (TextMetrics.wordErrorRate(reference: item.truth, hypothesis: page.plainText) * 10000).rounded() / 10000124                row["seconds_\(mode.rawValue)"] = (page.duration * 1000).rounded() / 1000125            }126            rows.append(row)127        }128        try write("ocr-accuracy.json", rows)129        #expect(rows.count == Corpus.all().count)130    }131132    @Test func timings() async throws {133        var results: [String: Any] = [:]134        let engine = OCREngine()135        let a4at200 = Fixture.page(Corpus.dense + "\n" + Fixture.english + "\n" + Fixture.french, fontSize: 30)136        let a4at300 = Fixture.page(Corpus.dense + "\n" + Fixture.english + "\n" + Fixture.french, width: 2480, height: 3508, fontSize: 45, margin: 210)137        _ = try await engine.recognize(a4at200, options: OCROptions(mode: .accurate))   // warm-up: model load is not part of the per-page figure138        for (label, image) in [("a4_200dpi", a4at200), ("a4_300dpi", a4at300)] {139            for mode in [RecognitionMode.accurate, .fast] {140                var times: [Double] = []141                for _ in 0..<5 { times.append(try await engine.recognize(image, options: OCROptions(mode: mode, languages: ["en-US", "fr-FR"])).duration) }142                results["ocr_\(mode.rawValue)_\(label)_median_s"] = (median(times) * 1000).rounded() / 1000143            }144        }145146        // Rendering: a 12-megapixel capture through each filter, full resolution, including JPEG encoding.147        let renderer = PageRenderer()148        let capture = Fixture.page(Corpus.dense, width: 3024, height: 4032, fontSize: 70, margin: 260, background: 0.82)149        let source = CIImage(cgImage: capture)150        for filter in FilterMode.allCases {151            var times: [Double] = []152            for _ in 0..<5 {153                let start = Date()154                _ = try renderer.jpegData(renderer.render(source, edit: PageEdit(filter: filter)), quality: 0.8)155                times.append(Date().timeIntervalSince(start))156            }157            results["render_12mp_\(filter.rawValue)_median_s"] = (median(times) * 1000).rounded() / 1000158        }159        var previewTimes: [Double] = []160        for _ in 0..<5 {161            let start = Date()162            _ = try renderer.cgImage(renderer.render(source, edit: PageEdit(filter: .color), maxDimension: 1800))163            previewTimes.append(Date().timeIntervalSince(start))164        }165        results["preview_1800px_color_median_s"] = (median(previewTimes) * 1000).rounded() / 1000166167        // Batch: a 25-page document recognized, then exported with each preset.168        let processor = Fixture.processor()169        // The document is built page by page so the test's own fixture images are gone before memory is sampled.170        var draft = try await processor.store.beginDraft(title: "Batch")171        for n in 0..<25 {172            let data = autoreleasepool { Fixture.jpeg(Fixture.page("Page \(n + 1)\n" + Corpus.dense, width: 2480, height: 3508, fontSize: 45, margin: 210)) }173            try await processor.store.addImage(data, fileExtension: "jpg", to: &draft)174        }175        let manifest = try await processor.store.commit(draft)176        let before = residentMegabytes()177        var start = Date()178        let summary = try await processor.recognize(manifest.id, options: OCROptions(mode: .accurate, languages: ["en-US"]))179        results["batch_25_pages_recognize_s"] = (Date().timeIntervalSince(start) * 100).rounded() / 100180        results["batch_25_pages_lines"] = summary.lineCount181        results["memory_after_recognize_25_pages_growth_mb"] = (residentMegabytes() - before).rounded()182        for preset in ExportPreset.allCases {183            start = Date()184            let export = try await processor.exportPDF(manifest.id, options: ExportOptions(preset: preset))185            results["export_25_pages_\(preset.rawValue)_s"] = (Date().timeIntervalSince(start) * 100).rounded() / 100186            results["export_25_pages_\(preset.rawValue)_bytes"] = export.byteSize187        }188        results["memory_after_recognize_and_3_exports_growth_mb"] = (residentMegabytes() - before).rounded()189        results["memory_baseline_before_batch_mb"] = before.rounded()190        results["stored_25_pages_bytes"] = await processor.store.byteSize(of: manifest.id)191        start = Date()192        _ = try await processor.exportText(manifest.id)193        results["export_text_25_pages_s"] = (Date().timeIntervalSince(start) * 1000).rounded() / 1000194195        var host: [String: Any] = ["os": ProcessInfo.processInfo.operatingSystemVersionString, "cores": ProcessInfo.processInfo.activeProcessorCount,196                                   "memory_gb": ProcessInfo.processInfo.physicalMemory / 1_073_741_824]197        var size = 0198        sysctlbyname("machdep.cpu.brand_string", nil, &size, nil, 0)199        var brand = [CChar](repeating: 0, count: max(size, 1))200        sysctlbyname("machdep.cpu.brand_string", &brand, &size, nil, 0)201        host["cpu"] = String(cString: brand)202        results["host"] = host203        try write("timings.json", results)204        #expect(summary.pagesRecognized == 25)205    }206}207
Files main
Menu

Theme