Repository Public License
1// ORIX Scan2// File: Packages/ORIXKit/Tests/ORIXKitTests/BenchmarkTests.swift3// Description: OCR accuracy benchmark (CER / WER on a synthetic corpus with known ground truth) and engine timings.4// Runs only when ORIX_BENCH is set; writes JSON to ORIX_BENCH_OUT. See Scripts/benchmark.sh.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import CoreGraphics9import CoreImage10import Foundation11import Testing12@testable import ORIXKit1314private let benchEnabled = ProcessInfo.processInfo.environment["ORIX_BENCH"] != nil1516struct CorpusCase {17 var name: String18 var category: String19 var truth: String20 var image: CGImage21}2223enum Corpus {24 static let mixed = """25 Meeting notes / Notes de réunion26 The budget was approved on Tuesday after a long discussion.27 Le budget a été approuvé mardi après une longue discussion.28 Next steps: hire two engineers and open the Québec office.29 Prochaines étapes : embaucher deux ingénieurs à Montréal.30 """3132 static let receipt = """33 CAFE DES ARTISANS34 87 rue Saint-Jean35 Latte grand 5.7536 Croissant beurre 3.9537 Sandwich jambon 9.2538 Sous-total 18.9539 Taxes 2.8440 TOTAL 21.7941 """4243 static let dense = """44 Section 4.2 describes the retention policy that applies to scanned records.45 Documents are stored locally and are never transmitted to a remote server.46 Each page keeps its original capture, so every edit can be reverted later.47 Recognition quality depends on resolution, contrast, focus and lighting.48 Tables are rebuilt from the position of words and may need a manual check.49 Exported files embed the image and an invisible, selectable text layer.50 """5152 static func lighting(_ image: CGImage) -> CGImage {53 // A strong diagonal shadow, as from a hand or a lamp at the side.54 let ctx = CGContext(data: nil, width: image.width, height: image.height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpace(name: CGColorSpace.sRGB)!,55 bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!56 ctx.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))57 let colors = [CGColor(red: 0, green: 0, blue: 0, alpha: 0), CGColor(red: 0, green: 0, blue: 0, alpha: 0.62)] as CFArray58 let gradient = CGGradient(colorsSpace: CGColorSpace(name: CGColorSpace.sRGB)!, colors: colors, locations: [0, 1])!59 ctx.drawLinearGradient(gradient, start: CGPoint(x: 0, y: CGFloat(image.height)), end: CGPoint(x: CGFloat(image.width), y: 0), options: [])60 return ctx.makeImage()!61 }6263 static func twoColumns() -> (CGImage, String) {64 let left = ["The left column comes first", "and continues on this line", "before the reader moves on."]65 let right = ["The right column follows", "with its own three lines", "that end the page."]66 let rows = (0..<3).map { [left[$0], right[$0]] }67 return (Fixture.table(rows, columnX: [120, 900], width: 1654, height: 700, fontSize: 34), (left + right).joined(separator: "\n"))68 }6970 static func all() -> [CorpusCase] {71 let (columns, columnsTruth) = twoColumns()72 return [73 CorpusCase(name: "English, clean print", category: "English", truth: Fixture.english, image: Fixture.page(Fixture.english)),74 CorpusCase(name: "English, dense 10 pt-equivalent", category: "English", truth: dense, image: Fixture.page(dense, fontSize: 28)),75 CorpusCase(name: "English, serif (Times)", category: "English", truth: Fixture.english, image: Fixture.page(Fixture.english, fontName: "Times New Roman")),76 CorpusCase(name: "French, clean print", category: "French", truth: Fixture.french, image: Fixture.page(Fixture.french)),77 CorpusCase(name: "French, serif (Georgia)", category: "French", truth: Fixture.french, image: Fixture.page(Fixture.french, fontName: "Georgia")),78 CorpusCase(name: "Mixed English / French", category: "Mixed", truth: mixed, image: Fixture.page(mixed)),79 CorpusCase(name: "Receipt, monospaced, narrow", category: "Receipt", truth: receipt,80 image: Fixture.page(receipt, width: 800, height: 1500, fontSize: 34, fontName: "Courier", margin: 70)),81 CorpusCase(name: "Low resolution (75 dpi-equivalent)", category: "Low resolution", truth: Fixture.english,82 image: Fixture.page(Fixture.english, width: 620, height: 877, fontSize: 15, margin: 52)),83 CorpusCase(name: "Skewed 3°", category: "Skew", truth: Fixture.english, image: Fixture.page(Fixture.english, rotationDegrees: 3)),84 CorpusCase(name: "Skewed 8°", category: "Skew", truth: Fixture.english, image: Fixture.page(Fixture.english, rotationDegrees: 8)),85 CorpusCase(name: "Two columns", category: "Multi-column", truth: columnsTruth, image: columns),86 CorpusCase(name: "Strong side shadow", category: "Difficult lighting", truth: Fixture.french, image: lighting(Fixture.page(Fixture.french, background: 0.9))),87 CorpusCase(name: "Dark page", category: "Difficult lighting", truth: Fixture.english, image: Fixture.page(Fixture.english, background: 0.38, ink: 0.04)),88 CorpusCase(name: "Faint ink", category: "Difficult lighting", truth: Fixture.english, image: Fixture.page(Fixture.english, background: 0.9, ink: 0.66)),89 CorpusCase(name: "Out of focus (blur σ 2.5)", category: "Blur", truth: Fixture.english, image: Fixture.blurred(Fixture.page(Fixture.english), radius: 2.5)),90 CorpusCase(name: "Script font (Snell Roundhand)", category: "Handwriting-like", truth: Fixture.english, image: Fixture.page(Fixture.english, fontSize: 48, fontName: "Snell Roundhand")),91 CorpusCase(name: "Casual script (Bradley Hand)", category: "Handwriting-like", truth: Fixture.french, image: Fixture.page(Fixture.french, fontSize: 44, fontName: "Bradley Hand")),92 ]93 }94}9596@Suite(.enabled(if: benchEnabled), .serialized) struct Benchmarks {97 private func write(_ name: String, _ object: Any) throws {98 guard let out = ProcessInfo.processInfo.environment["ORIX_BENCH_OUT"] else { return }99 try FileManager.default.createDirectory(atPath: out, withIntermediateDirectories: true)100 let data = try JSONSerialization.data(withJSONObject: object, options: [.prettyPrinted, .sortedKeys])101 try data.write(to: URL(fileURLWithPath: out).appendingPathComponent(name))102 }103104 private func median(_ values: [Double]) -> Double { values.sorted()[values.count / 2] }105106 private func residentMegabytes() -> Double {107 var info = task_vm_info_data_t()108 var count = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)109 let result = withUnsafeMutablePointer(to: &info) { $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) } }110 return result == KERN_SUCCESS ? Double(info.phys_footprint) / 1_048_576 : 0111 }112113 @Test func ocrAccuracy() async throws {114 var rows: [[String: Any]] = []115 let engine = OCREngine()116 for item in Corpus.all() {117 var row: [String: Any] = ["name": item.name, "category": item.category, "characters": TextMetrics.normalize(item.truth).count,118 "pixels": "\(item.image.width)x\(item.image.height)"]119 for mode in [RecognitionMode.automatic, .accurate, .fast] {120 let options = OCROptions(mode: mode, languages: mode == .automatic ? [] : ["en-US", "fr-FR"])121 let page = try await engine.recognize(item.image, options: options)122 row["cer_\(mode.rawValue)"] = (TextMetrics.characterErrorRate(reference: item.truth, hypothesis: page.plainText) * 10000).rounded() / 10000123 row["wer_\(mode.rawValue)"] = (TextMetrics.wordErrorRate(reference: item.truth, hypothesis: page.plainText) * 10000).rounded() / 10000124 row["seconds_\(mode.rawValue)"] = (page.duration * 1000).rounded() / 1000125 }126 rows.append(row)127 }128 try write("ocr-accuracy.json", rows)129 #expect(rows.count == Corpus.all().count)130 }131132 @Test func timings() async throws {133 var results: [String: Any] = [:]134 let engine = OCREngine()135 let a4at200 = Fixture.page(Corpus.dense + "\n" + Fixture.english + "\n" + Fixture.french, fontSize: 30)136 let a4at300 = Fixture.page(Corpus.dense + "\n" + Fixture.english + "\n" + Fixture.french, width: 2480, height: 3508, fontSize: 45, margin: 210)137 _ = try await engine.recognize(a4at200, options: OCROptions(mode: .accurate)) // warm-up: model load is not part of the per-page figure138 for (label, image) in [("a4_200dpi", a4at200), ("a4_300dpi", a4at300)] {139 for mode in [RecognitionMode.accurate, .fast] {140 var times: [Double] = []141 for _ in 0..<5 { times.append(try await engine.recognize(image, options: OCROptions(mode: mode, languages: ["en-US", "fr-FR"])).duration) }142 results["ocr_\(mode.rawValue)_\(label)_median_s"] = (median(times) * 1000).rounded() / 1000143 }144 }145146 // Rendering: a 12-megapixel capture through each filter, full resolution, including JPEG encoding.147 let renderer = PageRenderer()148 let capture = Fixture.page(Corpus.dense, width: 3024, height: 4032, fontSize: 70, margin: 260, background: 0.82)149 let source = CIImage(cgImage: capture)150 for filter in FilterMode.allCases {151 var times: [Double] = []152 for _ in 0..<5 {153 let start = Date()154 _ = try renderer.jpegData(renderer.render(source, edit: PageEdit(filter: filter)), quality: 0.8)155 times.append(Date().timeIntervalSince(start))156 }157 results["render_12mp_\(filter.rawValue)_median_s"] = (median(times) * 1000).rounded() / 1000158 }159 var previewTimes: [Double] = []160 for _ in 0..<5 {161 let start = Date()162 _ = try renderer.cgImage(renderer.render(source, edit: PageEdit(filter: .color), maxDimension: 1800))163 previewTimes.append(Date().timeIntervalSince(start))164 }165 results["preview_1800px_color_median_s"] = (median(previewTimes) * 1000).rounded() / 1000166167 // Batch: a 25-page document recognized, then exported with each preset.168 let processor = Fixture.processor()169 // The document is built page by page so the test's own fixture images are gone before memory is sampled.170 var draft = try await processor.store.beginDraft(title: "Batch")171 for n in 0..<25 {172 let data = autoreleasepool { Fixture.jpeg(Fixture.page("Page \(n + 1)\n" + Corpus.dense, width: 2480, height: 3508, fontSize: 45, margin: 210)) }173 try await processor.store.addImage(data, fileExtension: "jpg", to: &draft)174 }175 let manifest = try await processor.store.commit(draft)176 let before = residentMegabytes()177 var start = Date()178 let summary = try await processor.recognize(manifest.id, options: OCROptions(mode: .accurate, languages: ["en-US"]))179 results["batch_25_pages_recognize_s"] = (Date().timeIntervalSince(start) * 100).rounded() / 100180 results["batch_25_pages_lines"] = summary.lineCount181 results["memory_after_recognize_25_pages_growth_mb"] = (residentMegabytes() - before).rounded()182 for preset in ExportPreset.allCases {183 start = Date()184 let export = try await processor.exportPDF(manifest.id, options: ExportOptions(preset: preset))185 results["export_25_pages_\(preset.rawValue)_s"] = (Date().timeIntervalSince(start) * 100).rounded() / 100186 results["export_25_pages_\(preset.rawValue)_bytes"] = export.byteSize187 }188 results["memory_after_recognize_and_3_exports_growth_mb"] = (residentMegabytes() - before).rounded()189 results["memory_baseline_before_batch_mb"] = before.rounded()190 results["stored_25_pages_bytes"] = await processor.store.byteSize(of: manifest.id)191 start = Date()192 _ = try await processor.exportText(manifest.id)193 results["export_text_25_pages_s"] = (Date().timeIntervalSince(start) * 1000).rounded() / 1000194195 var host: [String: Any] = ["os": ProcessInfo.processInfo.operatingSystemVersionString, "cores": ProcessInfo.processInfo.activeProcessorCount,196 "memory_gb": ProcessInfo.processInfo.physicalMemory / 1_073_741_824]197 var size = 0198 sysctlbyname("machdep.cpu.brand_string", nil, &size, nil, 0)199 var brand = [CChar](repeating: 0, count: max(size, 1))200 sysctlbyname("machdep.cpu.brand_string", &brand, &size, nil, 0)201 host["cpu"] = String(cString: brand)202 results["host"] = host203 try write("timings.json", results)204 #expect(summary.pagesRecognized == 25)205 }206}207