Repository Public
1// SP Notes iOS2// File: Tests/InkHandwritingTests.swift3// Description: Handwriting → text: which strokes a region selects, the picture made of only those strokes, the mapping of the4// recogniser's candidates and confidences, the Markdown offered, and what « Insérer » / « Remplacer » do to the5// drawing. Vision is exercised on strokes that trace typed letters (real handwriting needs a device and a Pencil).6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import PencilKit10import Testing11import UIKit12import SPNotesCore13@testable import SPNotes1415/// A recogniser that answers what it is told and remembers what it was shown.16final class ScriptedRecognizer: HandwritingRecognizing, @unchecked Sendable {17 private let lock = NSLock()18 private var seen: [CGImage] = []19 let lines: [HandwritingLine]20 init(_ lines: [HandwritingLine]) { self.lines = lines }21 func recognize(_ image: CGImage) async -> [HandwritingLine] {22 lock.withLock { seen.append(image) }23 return lines24 }25 var images: [CGImage] { lock.withLock { seen } }26}2728@MainActor29@Suite("Ink — écriture manuscrite → texte") struct InkHandwritingTests {30 func twoWords() -> PKDrawing {31 PKDrawing(strokes: TypedStrokes.strokes("BONJOUR", at: CGPoint(x: 60, y: 140)) + TypedStrokes.strokes("PENCIL", at: CGPoint(x: 60, y: 420)))32 }3334 @Test func aRegionSelectsTheStrokesInsideIt() {35 let drawing = twoWords()36 let upper = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 40, width: 700, height: 140))37 let lower = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 330, width: 700, height: 140))38 #expect(!upper.isEmpty && !lower.isEmpty)39 #expect(Set(upper).isDisjoint(with: lower))40 #expect(upper.count + lower.count == drawing.strokes.count)41 #expect(HandwritingPipeline.strokeIndices(of: drawing, in: nil).count == drawing.strokes.count)42 #expect(HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 900, y: 900, width: 50, height: 50)).isEmpty)43 // a stroke that only brushes the frame is not taken44 #expect(HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 0, width: 700, height: 96)).count < upper.count)45 #expect(HandwritingPipeline.drawing(drawing, keeping: upper).strokes.count == upper.count)46 #expect(HandwritingPipeline.drawing(drawing, removing: upper).strokes.count == lower.count)47 }4849 @Test func thePictureHoldsOnlyTheSelectedStrokesOnWhite() throws {50 let drawing = twoWords()51 let upper = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 40, width: 700, height: 140))52 let image = try #require(HandwritingPipeline.bitmap(of: drawing, indices: upper))53 let bounds = HandwritingPipeline.drawing(drawing, keeping: upper).bounds54 // cropped to the selection (not the whole drawing), at a high scale55 let scale = CGFloat(image.width) / (bounds.width + 2 * HandwritingPipeline.margin)56 #expect(scale >= 2.9 && scale <= 4.1)57 #expect(abs(CGFloat(image.height) / scale - (bounds.height + 2 * HandwritingPipeline.margin)) <= 3)58 #expect(CGFloat(image.height) / scale < 200) // the second word, 280 pt lower, is not in the picture59 // white paper, dark ink (read through an 8-bit RGBA copy, whatever the format of the picture)60 let width = image.width, height = image.height61 var pixels = [UInt8](repeating: 0, count: width * height * 4)62 let context = try #require(CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))63 context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))64 var dark = 0, light = 065 for y in stride(from: 0, to: height, by: 7) {66 for x in stride(from: 0, to: width, by: 7) {67 let offset = (y * width + x) * 468 let sum = Int(pixels[offset]) + Int(pixels[offset + 1]) + Int(pixels[offset + 2])69 if sum < 200 { dark += 1 } else if sum > 700 { light += 1 }70 }71 }72 #expect(image.bitsPerComponent == 8) // standard range: no 16-bit pictures for Vision or for the previews73 #expect(dark > 20 && light > dark * 3)74 #expect(HandwritingPipeline.bitmap(of: drawing, indices: []) == nil)75 }7677 @Test func candidatesAndConfidencesBecomeAnOrderedResult() async throws {78 let drawing = twoWords()79 let recognizer = ScriptedRecognizer([80 HandwritingLine(text: " # deuxième ligne ", confidence: 0.5, box: CGRect(x: 0.1, y: 0.6, width: 0.8, height: 0.2)),81 HandwritingLine(text: "", confidence: 0.9, box: CGRect(x: 0, y: 0.4, width: 0.1, height: 0.1)),82 HandwritingLine(text: "Première", confidence: 1.4, box: CGRect(x: 0.1, y: 0.1, width: 0.6, height: 0.2)),83 ])84 let region = CGRect(x: 0, y: 40, width: 700, height: 140)85 let proposal = try #require(await HandwritingPipeline.recognize(drawing, region: region, recognizer: recognizer))86 #expect(proposal.strokeIndices == HandwritingPipeline.strokeIndices(of: drawing, in: region))87 #expect(recognizer.images.count == 1)88 #expect(proposal.result.lines.map(\.text) == ["Première", "# deuxième ligne"])89 #expect(proposal.result.text == "Première\n# deuxième ligne")90 // weighted by length: (1.0 × 8 + 0.5 × 16) / 2491 #expect(abs(proposal.result.confidence - 16.0 / 24.0) < 1e-9)92 #expect(proposal.result.confidencePercent == 67)93 // what would be inserted is text, never a heading94 #expect(proposal.result.markdown == "Première\n\n\\# deuxième ligne")95 let inserted = NoteDocument(markdown: proposal.result.markdown)96 #expect(inserted.lines.map(\.type) == ["paragraph", "paragraph"])97 #expect(inserted.plainText().contains("# deuxième ligne"))98 // an empty region gives no proposal and the recogniser is not called99 #expect(await HandwritingPipeline.recognize(drawing, region: CGRect(x: 900, y: 900, width: 10, height: 10), recognizer: recognizer) == nil)100 #expect(recognizer.images.count == 1)101 #expect(HandwritingResult(lines: []).confidence == 0 && HandwritingResult(lines: []).isEmpty)102 }103104 @Test func markdownCharactersAreEscaped() {105 for text in ["- liste", "+ plus", "1. un", "12) douze", "*gras* et _it_", "a < b > c", "`code` [lien] | ~x~", "prix : 3 * 4 # 5", "\\"] {106 let document = NoteDocument(markdown: HandwritingResult.escape(text))107 #expect(document.lines.count == 1 && document.lines[0].type == "paragraph", "\(text) → \(document.lines.map(\.type))")108 #expect(document.plainText().trimmingCharacters(in: .whitespacesAndNewlines) == text, "\(text) → \(document.plainText())")109 }110 }111112 @Test func visionBoxesAreFlippedToTheTopLeft() {113 let line = VisionHandwritingRecognizer.line(text: "abc", confidence: 0.8, visionBox: CGRect(x: 0.1, y: 0.7, width: 0.5, height: 0.2))114 #expect(line?.box.minY ?? 0 > 0.099 && line?.box.minY ?? 1 < 0.101)115 #expect(abs((line?.confidence ?? 0) - 0.8) < 1e-6)116 #expect(VisionHandwritingRecognizer.line(text: nil, confidence: 1, visionBox: .zero) == nil)117 #expect(VisionHandwritingRecognizer.line(text: "", confidence: 1, visionBox: .zero) == nil)118 }119120 @Test func insertQueuesTheTextAndReplaceAlsoRemovesTheStrokes() throws {121 let scene = try #require(UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.first)122 let window = UIWindow(windowScene: scene)123 window.frame = CGRect(x: 0, y: 0, width: 900, height: 700)124 defer { window.isHidden = true }125 let model = DrawingModel()126 let drawing = twoWords()127 let controller = DrawingCanvasController(model: model, initial: drawing, fingerDrawing: true)128 window.rootViewController = controller129 window.isHidden = false130 controller.loadViewIfNeeded()131 let upper = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 40, width: 700, height: 140))132 let proposal = HandwritingProposal(strokeIndices: upper, result: HandwritingResult(lines: [HandwritingLine(text: "Bonjour", confidence: 0.9, box: .zero)]))133134 // nothing happens until the user answers135 model.proposal = proposal136 #expect(model.pendingInsertions.isEmpty && model.drawing.strokes.count == drawing.strokes.count)137 // « Insérer sous le dessin »: the text waits for "Terminé", the drawing is untouched138 model.accept(proposal, replacingStrokes: false)139 #expect(model.pendingInsertions == ["Bonjour"] && model.proposal == nil)140 #expect(model.drawing == drawing)141 #expect(model.isDirty)142 // « Remplacer le dessin sélectionné »: the strokes that were read are removed, as one undoable step143 model.accept(proposal, replacingStrokes: true)144 #expect(model.pendingInsertions == ["Bonjour", "Bonjour"])145 #expect(model.drawing.strokes.count == drawing.strokes.count - upper.count)146 #expect(model.drawing.strokes.map(\.renderBounds) == HandwritingPipeline.drawing(drawing, removing: upper).strokes.map(\.renderBounds))147 controller.canvas.undoManager?.undo()148 #expect(model.drawing == drawing)149 // an empty recognition inserts nothing150 model.accept(HandwritingProposal(strokeIndices: upper, result: HandwritingResult(lines: [])), replacingStrokes: true)151 #expect(model.pendingInsertions.count == 2 && model.drawing == drawing)152 }153154 /// Vision, for real, on strokes tracing typed capitals. Handwriting itself can only be judged on a device with a Pencil.155 @Test func visionReadsStrokesThatTraceTypedLetters() async throws {156 let drawing = twoWords()157 let region = CGRect(x: 0, y: 40, width: 700, height: 140)158 let proposal = try #require(await HandwritingPipeline.recognize(drawing, region: region, recognizer: VisionHandwritingRecognizer()))159 try? "vision: '\(proposal.result.text)' confidence \(proposal.result.confidence)\n".write(toFile: "/tmp/spnote-next/pencil-work/vision.txt", atomically: true, encoding: .utf8)160 #expect(proposal.result.text.uppercased().contains("BONJOUR"))161 #expect(!proposal.result.text.uppercased().contains("PENCIL")) // only the framed strokes were read162 #expect(proposal.result.confidence > 0.3 && proposal.result.confidence <= 1)163 }164}165