SPB Git

Repository Public

HandwritingRecognition.swift 7.9 KB 162 lines swift

1// SP Notes iOS2// File: Core/Ink/HandwritingRecognition.swift3// Description: Handwriting → text for a part of a drawing: picks the strokes of a region, renders only those on white at a high4//              scale, runs an on-device recogniser (Vision, French + English) and returns the lines with their confidence. The5//              result is only ever offered to the user; nothing here writes into a note.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import PencilKit10import UIKit11import Vision1213/// One recognised line. `box` is in the unit square of the recognised picture, origin top-left.14struct HandwritingLine: Equatable, Sendable {15    var text: String16    /// 0…1, as reported by the recogniser.17    var confidence: Double18    var box: CGRect19}2021struct HandwritingResult: Equatable, Sendable {22    var lines: [HandwritingLine]2324    /// Lines top to bottom (left to right on the same row), empty ones dropped.25    init(lines: [HandwritingLine]) {26        self.lines = lines27            .map { HandwritingLine(text: $0.text.trimmingCharacters(in: .whitespacesAndNewlines), confidence: min(1, max(0, $0.confidence)), box: $0.box) }28            .filter { !$0.text.isEmpty }29            .sorted { a, b in abs(a.box.midY - b.box.midY) > min(a.box.height, b.box.height) / 2 ? a.box.midY < b.box.midY : a.box.minX < b.box.minX }30    }3132    var isEmpty: Bool { lines.isEmpty }33    var text: String { lines.map(\.text).joined(separator: "\n") }3435    /// Average confidence, each line weighted by its length. 0 when nothing was read.36    var confidence: Double {37        let weight = lines.reduce(0) { $0 + $1.text.count }38        guard weight > 0 else { return 0 }39        return lines.reduce(0) { $0 + $1.confidence * Double($1.text.count) } / Double(weight)40    }4142    /// 0…100, for display.43    var confidencePercent: Int { Int((confidence * 100).rounded()) }4445    /// The text as Markdown paragraphs (one per line), with the characters Markdown would interpret escaped: what is inserted46    /// is what was read, never a heading or a list by accident.47    var markdown: String { lines.map { Self.escape($0.text) }.joined(separator: "\n\n") }4849    static func escape(_ line: String) -> String {50        var out = ""51        for character in line {52            if "\\`*_[]<>#|~".contains(character) { out.append("\\") }53            out.append(character)54        }55        // a line that would start a list, a quote or a numbered item56        if let first = out.first, "-+".contains(first), out.dropFirst().first == " " { out = "\\" + out }57        if let match = out.range(of: "^[0-9]+[.)] ", options: .regularExpression) {58            let digits = out[match].dropLast(2)59            out = digits + "\\" + out[out.index(match.lowerBound, offsetBy: digits.count)...]60        }61        return out62    }63}6465/// What reads a picture. The app uses Vision; tests inject their own.66protocol HandwritingRecognizing: Sendable {67    func recognize(_ image: CGImage) async -> [HandwritingLine]68}6970/// On-device recognition with Vision: accurate level, language correction, French and English.71struct VisionHandwritingRecognizer: HandwritingRecognizing {72    static let preferredLanguages = ["fr-FR", "fr-CA", "en-US"]7374    func recognize(_ image: CGImage) async -> [HandwritingLine] {75        await Task.detached(priority: .userInitiated) {76            let request = VNRecognizeTextRequest()77            request.recognitionLevel = .accurate78            request.usesLanguageCorrection = true79            if let supported = try? request.supportedRecognitionLanguages() {80                let languages = Self.preferredLanguages.filter(supported.contains)81                if !languages.isEmpty { request.recognitionLanguages = languages }82            }83            do { try VNImageRequestHandler(cgImage: image, orientation: .up).perform([request]) } catch { return [] }84            return (request.results ?? []).compactMap { observation in85                Self.line(text: observation.topCandidates(1).first?.string, confidence: observation.topCandidates(1).first?.confidence, visionBox: observation.boundingBox)86            }87        }.value88    }8990    /// Vision boxes have their origin at the bottom-left: flipped to top-left here.91    static func line(text: String?, confidence: Float?, visionBox: CGRect) -> HandwritingLine? {92        guard let text, !text.isEmpty else { return nil }93        return HandwritingLine(text: text, confidence: Double(confidence ?? 0), box: CGRect(x: visionBox.minX, y: 1 - visionBox.maxY, width: visionBox.width, height: visionBox.height))94    }95}9697/// A recognition offered to the user: which strokes were read, and what was read.98struct HandwritingProposal: Identifiable, Equatable, Sendable {99    let id = UUID()100    /// Indices of the strokes in the drawing the proposal was made from.101    var strokeIndices: [Int]102    var result: HandwritingResult103    static func == (a: Self, b: Self) -> Bool { a.id == b.id }104}105106enum HandwritingPipeline {107    static let margin: CGFloat = 16108    static let maximumPixels: CGFloat = 4096109110    /// The strokes to read: all of them without a region; with one, those whose ink lies mostly inside it (its centre in the111    /// region, or at least half of its box).112    static func strokeIndices(of drawing: PKDrawing, in region: CGRect?) -> [Int] {113        guard let region else { return Array(drawing.strokes.indices) }114        return drawing.strokes.indices.filter { index in115            let box = drawing.strokes[index].renderBounds116            guard !box.isNull, box.intersects(region) else { return false }117            if region.contains(CGPoint(x: box.midX, y: box.midY)) { return true }118            let shared = box.intersection(region), area = max(box.width * box.height, 1)119            return shared.width * shared.height / area >= 0.5120        }121    }122123    static func drawing(_ drawing: PKDrawing, keeping indices: [Int]) -> PKDrawing {124        let wanted = Set(indices)125        return PKDrawing(strokes: drawing.strokes.enumerated().filter { wanted.contains($0.offset) }.map(\.element))126    }127128    static func drawing(_ drawing: PKDrawing, removing indices: [Int]) -> PKDrawing {129        let unwanted = Set(indices)130        return PKDrawing(strokes: drawing.strokes.enumerated().filter { !unwanted.contains($0.offset) }.map(\.element))131    }132133    /// Only the chosen strokes, on white, as in light mode, at the highest scale (up to 4×) that keeps the picture under 4096 px.134    @MainActor135    static func bitmap(of drawing: PKDrawing, indices: [Int]) -> CGImage? {136        let selection = self.drawing(drawing, keeping: indices)137        let bounds = selection.bounds138        guard !selection.strokes.isEmpty, !bounds.isNull, !bounds.isInfinite, bounds.width > 0 || bounds.height > 0 else { return nil }139        let rect = bounds.insetBy(dx: -margin, dy: -margin).integral140        let scale = max(1, min(4, maximumPixels / max(rect.width, rect.height)))141        var ink: UIImage?142        UITraitCollection(userInterfaceStyle: .light).performAsCurrent { ink = selection.image(from: rect, scale: scale) }143        let format = UIGraphicsImageRendererFormat()144        format.scale = scale145        format.opaque = true146        format.preferredRange = .standard147        return UIGraphicsImageRenderer(size: rect.size, format: format).image { context in148            UIColor.white.setFill()149            context.fill(CGRect(origin: .zero, size: rect.size))150            ink?.draw(in: CGRect(origin: .zero, size: rect.size))151        }.cgImage152    }153154    /// Selection → picture → recogniser → proposal. Nil when the region holds no stroke.155    @MainActor156    static func recognize(_ drawing: PKDrawing, region: CGRect?, recognizer: any HandwritingRecognizing) async -> HandwritingProposal? {157        let indices = strokeIndices(of: drawing, in: region)158        guard !indices.isEmpty, let image = bitmap(of: drawing, indices: indices) else { return nil }159        return HandwritingProposal(strokeIndices: indices, result: HandwritingResult(lines: await recognizer.recognize(image)))160    }161}162
Files main
Menu

Theme