SPB Git

Repository Public

InkHandwritingTests.swift 10.4 KB 165 lines swift

1// SP Notes iOS2// File: Tests/InkHandwritingTests.swift3// Description: Handwriting → text: which strokes a region selects, the picture made of only those strokes, the mapping of the4//              recogniser's candidates and confidences, the Markdown offered, and what « Insérer » / « Remplacer » do to the5//              drawing. Vision is exercised on strokes that trace typed letters (real handwriting needs a device and a Pencil).6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import PencilKit10import Testing11import UIKit12import SPNotesCore13@testable import SPNotes1415/// A recogniser that answers what it is told and remembers what it was shown.16final class ScriptedRecognizer: HandwritingRecognizing, @unchecked Sendable {17    private let lock = NSLock()18    private var seen: [CGImage] = []19    let lines: [HandwritingLine]20    init(_ lines: [HandwritingLine]) { self.lines = lines }21    func recognize(_ image: CGImage) async -> [HandwritingLine] {22        lock.withLock { seen.append(image) }23        return lines24    }25    var images: [CGImage] { lock.withLock { seen } }26}2728@MainActor29@Suite("Ink — écriture manuscrite → texte") struct InkHandwritingTests {30    func twoWords() -> PKDrawing {31        PKDrawing(strokes: TypedStrokes.strokes("BONJOUR", at: CGPoint(x: 60, y: 140)) + TypedStrokes.strokes("PENCIL", at: CGPoint(x: 60, y: 420)))32    }3334    @Test func aRegionSelectsTheStrokesInsideIt() {35        let drawing = twoWords()36        let upper = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 40, width: 700, height: 140))37        let lower = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 330, width: 700, height: 140))38        #expect(!upper.isEmpty && !lower.isEmpty)39        #expect(Set(upper).isDisjoint(with: lower))40        #expect(upper.count + lower.count == drawing.strokes.count)41        #expect(HandwritingPipeline.strokeIndices(of: drawing, in: nil).count == drawing.strokes.count)42        #expect(HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 900, y: 900, width: 50, height: 50)).isEmpty)43        // a stroke that only brushes the frame is not taken44        #expect(HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 0, width: 700, height: 96)).count < upper.count)45        #expect(HandwritingPipeline.drawing(drawing, keeping: upper).strokes.count == upper.count)46        #expect(HandwritingPipeline.drawing(drawing, removing: upper).strokes.count == lower.count)47    }4849    @Test func thePictureHoldsOnlyTheSelectedStrokesOnWhite() throws {50        let drawing = twoWords()51        let upper = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 40, width: 700, height: 140))52        let image = try #require(HandwritingPipeline.bitmap(of: drawing, indices: upper))53        let bounds = HandwritingPipeline.drawing(drawing, keeping: upper).bounds54        // cropped to the selection (not the whole drawing), at a high scale55        let scale = CGFloat(image.width) / (bounds.width + 2 * HandwritingPipeline.margin)56        #expect(scale >= 2.9 && scale <= 4.1)57        #expect(abs(CGFloat(image.height) / scale - (bounds.height + 2 * HandwritingPipeline.margin)) <= 3)58        #expect(CGFloat(image.height) / scale < 200) // the second word, 280 pt lower, is not in the picture59        // white paper, dark ink (read through an 8-bit RGBA copy, whatever the format of the picture)60        let width = image.width, height = image.height61        var pixels = [UInt8](repeating: 0, count: width * height * 4)62        let context = try #require(CGContext(data: &pixels, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))63        context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))64        var dark = 0, light = 065        for y in stride(from: 0, to: height, by: 7) {66            for x in stride(from: 0, to: width, by: 7) {67                let offset = (y * width + x) * 468                let sum = Int(pixels[offset]) + Int(pixels[offset + 1]) + Int(pixels[offset + 2])69                if sum < 200 { dark += 1 } else if sum > 700 { light += 1 }70            }71        }72        #expect(image.bitsPerComponent == 8) // standard range: no 16-bit pictures for Vision or for the previews73        #expect(dark > 20 && light > dark * 3)74        #expect(HandwritingPipeline.bitmap(of: drawing, indices: []) == nil)75    }7677    @Test func candidatesAndConfidencesBecomeAnOrderedResult() async throws {78        let drawing = twoWords()79        let recognizer = ScriptedRecognizer([80            HandwritingLine(text: "  # deuxième ligne ", confidence: 0.5, box: CGRect(x: 0.1, y: 0.6, width: 0.8, height: 0.2)),81            HandwritingLine(text: "", confidence: 0.9, box: CGRect(x: 0, y: 0.4, width: 0.1, height: 0.1)),82            HandwritingLine(text: "Première", confidence: 1.4, box: CGRect(x: 0.1, y: 0.1, width: 0.6, height: 0.2)),83        ])84        let region = CGRect(x: 0, y: 40, width: 700, height: 140)85        let proposal = try #require(await HandwritingPipeline.recognize(drawing, region: region, recognizer: recognizer))86        #expect(proposal.strokeIndices == HandwritingPipeline.strokeIndices(of: drawing, in: region))87        #expect(recognizer.images.count == 1)88        #expect(proposal.result.lines.map(\.text) == ["Première", "# deuxième ligne"])89        #expect(proposal.result.text == "Première\n# deuxième ligne")90        // weighted by length: (1.0 × 8 + 0.5 × 16) / 2491        #expect(abs(proposal.result.confidence - 16.0 / 24.0) < 1e-9)92        #expect(proposal.result.confidencePercent == 67)93        // what would be inserted is text, never a heading94        #expect(proposal.result.markdown == "Première\n\n\\# deuxième ligne")95        let inserted = NoteDocument(markdown: proposal.result.markdown)96        #expect(inserted.lines.map(\.type) == ["paragraph", "paragraph"])97        #expect(inserted.plainText().contains("# deuxième ligne"))98        // an empty region gives no proposal and the recogniser is not called99        #expect(await HandwritingPipeline.recognize(drawing, region: CGRect(x: 900, y: 900, width: 10, height: 10), recognizer: recognizer) == nil)100        #expect(recognizer.images.count == 1)101        #expect(HandwritingResult(lines: []).confidence == 0 && HandwritingResult(lines: []).isEmpty)102    }103104    @Test func markdownCharactersAreEscaped() {105        for text in ["- liste", "+ plus", "1. un", "12) douze", "*gras* et _it_", "a < b > c", "`code` [lien] | ~x~", "prix : 3 * 4 # 5", "\\"] {106            let document = NoteDocument(markdown: HandwritingResult.escape(text))107            #expect(document.lines.count == 1 && document.lines[0].type == "paragraph", "\(text) → \(document.lines.map(\.type))")108            #expect(document.plainText().trimmingCharacters(in: .whitespacesAndNewlines) == text, "\(text) → \(document.plainText())")109        }110    }111112    @Test func visionBoxesAreFlippedToTheTopLeft() {113        let line = VisionHandwritingRecognizer.line(text: "abc", confidence: 0.8, visionBox: CGRect(x: 0.1, y: 0.7, width: 0.5, height: 0.2))114        #expect(line?.box.minY ?? 0 > 0.099 && line?.box.minY ?? 1 < 0.101)115        #expect(abs((line?.confidence ?? 0) - 0.8) < 1e-6)116        #expect(VisionHandwritingRecognizer.line(text: nil, confidence: 1, visionBox: .zero) == nil)117        #expect(VisionHandwritingRecognizer.line(text: "", confidence: 1, visionBox: .zero) == nil)118    }119120    @Test func insertQueuesTheTextAndReplaceAlsoRemovesTheStrokes() throws {121        let scene = try #require(UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.first)122        let window = UIWindow(windowScene: scene)123        window.frame = CGRect(x: 0, y: 0, width: 900, height: 700)124        defer { window.isHidden = true }125        let model = DrawingModel()126        let drawing = twoWords()127        let controller = DrawingCanvasController(model: model, initial: drawing, fingerDrawing: true)128        window.rootViewController = controller129        window.isHidden = false130        controller.loadViewIfNeeded()131        let upper = HandwritingPipeline.strokeIndices(of: drawing, in: CGRect(x: 0, y: 40, width: 700, height: 140))132        let proposal = HandwritingProposal(strokeIndices: upper, result: HandwritingResult(lines: [HandwritingLine(text: "Bonjour", confidence: 0.9, box: .zero)]))133134        // nothing happens until the user answers135        model.proposal = proposal136        #expect(model.pendingInsertions.isEmpty && model.drawing.strokes.count == drawing.strokes.count)137        // « Insérer sous le dessin »: the text waits for "Terminé", the drawing is untouched138        model.accept(proposal, replacingStrokes: false)139        #expect(model.pendingInsertions == ["Bonjour"] && model.proposal == nil)140        #expect(model.drawing == drawing)141        #expect(model.isDirty)142        // « Remplacer le dessin sélectionné »: the strokes that were read are removed, as one undoable step143        model.accept(proposal, replacingStrokes: true)144        #expect(model.pendingInsertions == ["Bonjour", "Bonjour"])145        #expect(model.drawing.strokes.count == drawing.strokes.count - upper.count)146        #expect(model.drawing.strokes.map(\.renderBounds) == HandwritingPipeline.drawing(drawing, removing: upper).strokes.map(\.renderBounds))147        controller.canvas.undoManager?.undo()148        #expect(model.drawing == drawing)149        // an empty recognition inserts nothing150        model.accept(HandwritingProposal(strokeIndices: upper, result: HandwritingResult(lines: [])), replacingStrokes: true)151        #expect(model.pendingInsertions.count == 2 && model.drawing == drawing)152    }153154    /// Vision, for real, on strokes tracing typed capitals. Handwriting itself can only be judged on a device with a Pencil.155    @Test func visionReadsStrokesThatTraceTypedLetters() async throws {156        let drawing = twoWords()157        let region = CGRect(x: 0, y: 40, width: 700, height: 140)158        let proposal = try #require(await HandwritingPipeline.recognize(drawing, region: region, recognizer: VisionHandwritingRecognizer()))159        try? "vision: '\(proposal.result.text)' confidence \(proposal.result.confidence)\n".write(toFile: "/tmp/spnote-next/pencil-work/vision.txt", atomically: true, encoding: .utf8)160        #expect(proposal.result.text.uppercased().contains("BONJOUR"))161        #expect(!proposal.result.text.uppercased().contains("PENCIL")) // only the framed strokes were read162        #expect(proposal.result.confidence > 0.3 && proposal.result.confidence <= 1)163    }164}165
Files main
Menu

Theme