Repository Public
1// SP Notes iOS2// File: Core/Ink/HandwritingRecognition.swift3// Description: Handwriting → text for a part of a drawing: picks the strokes of a region, renders only those on white at a high4// scale, runs an on-device recogniser (Vision, French + English) and returns the lines with their confidence. The5// result is only ever offered to the user; nothing here writes into a note.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import PencilKit10import UIKit11import Vision1213/// One recognised line. `box` is in the unit square of the recognised picture, origin top-left.14struct HandwritingLine: Equatable, Sendable {15 var text: String16 /// 0…1, as reported by the recogniser.17 var confidence: Double18 var box: CGRect19}2021struct HandwritingResult: Equatable, Sendable {22 var lines: [HandwritingLine]2324 /// Lines top to bottom (left to right on the same row), empty ones dropped.25 init(lines: [HandwritingLine]) {26 self.lines = lines27 .map { HandwritingLine(text: $0.text.trimmingCharacters(in: .whitespacesAndNewlines), confidence: min(1, max(0, $0.confidence)), box: $0.box) }28 .filter { !$0.text.isEmpty }29 .sorted { a, b in abs(a.box.midY - b.box.midY) > min(a.box.height, b.box.height) / 2 ? a.box.midY < b.box.midY : a.box.minX < b.box.minX }30 }3132 var isEmpty: Bool { lines.isEmpty }33 var text: String { lines.map(\.text).joined(separator: "\n") }3435 /// Average confidence, each line weighted by its length. 0 when nothing was read.36 var confidence: Double {37 let weight = lines.reduce(0) { $0 + $1.text.count }38 guard weight > 0 else { return 0 }39 return lines.reduce(0) { $0 + $1.confidence * Double($1.text.count) } / Double(weight)40 }4142 /// 0…100, for display.43 var confidencePercent: Int { Int((confidence * 100).rounded()) }4445 /// The text as Markdown paragraphs (one per line), with the characters Markdown would interpret escaped: what is inserted46 /// is what was read, never a heading or a list by accident.47 var markdown: String { lines.map { Self.escape($0.text) }.joined(separator: "\n\n") }4849 static func escape(_ line: String) -> String {50 var out = ""51 for character in line {52 if "\\`*_[]<>#|~".contains(character) { out.append("\\") }53 out.append(character)54 }55 // a line that would start a list, a quote or a numbered item56 if let first = out.first, "-+".contains(first), out.dropFirst().first == " " { out = "\\" + out }57 if let match = out.range(of: "^[0-9]+[.)] ", options: .regularExpression) {58 let digits = out[match].dropLast(2)59 out = digits + "\\" + out[out.index(match.lowerBound, offsetBy: digits.count)...]60 }61 return out62 }63}6465/// What reads a picture. The app uses Vision; tests inject their own.66protocol HandwritingRecognizing: Sendable {67 func recognize(_ image: CGImage) async -> [HandwritingLine]68}6970/// On-device recognition with Vision: accurate level, language correction, French and English.71struct VisionHandwritingRecognizer: HandwritingRecognizing {72 static let preferredLanguages = ["fr-FR", "fr-CA", "en-US"]7374 func recognize(_ image: CGImage) async -> [HandwritingLine] {75 await Task.detached(priority: .userInitiated) {76 let request = VNRecognizeTextRequest()77 request.recognitionLevel = .accurate78 request.usesLanguageCorrection = true79 if let supported = try? request.supportedRecognitionLanguages() {80 let languages = Self.preferredLanguages.filter(supported.contains)81 if !languages.isEmpty { request.recognitionLanguages = languages }82 }83 do { try VNImageRequestHandler(cgImage: image, orientation: .up).perform([request]) } catch { return [] }84 return (request.results ?? []).compactMap { observation in85 Self.line(text: observation.topCandidates(1).first?.string, confidence: observation.topCandidates(1).first?.confidence, visionBox: observation.boundingBox)86 }87 }.value88 }8990 /// Vision boxes have their origin at the bottom-left: flipped to top-left here.91 static func line(text: String?, confidence: Float?, visionBox: CGRect) -> HandwritingLine? {92 guard let text, !text.isEmpty else { return nil }93 return HandwritingLine(text: text, confidence: Double(confidence ?? 0), box: CGRect(x: visionBox.minX, y: 1 - visionBox.maxY, width: visionBox.width, height: visionBox.height))94 }95}9697/// A recognition offered to the user: which strokes were read, and what was read.98struct HandwritingProposal: Identifiable, Equatable, Sendable {99 let id = UUID()100 /// Indices of the strokes in the drawing the proposal was made from.101 var strokeIndices: [Int]102 var result: HandwritingResult103 static func == (a: Self, b: Self) -> Bool { a.id == b.id }104}105106enum HandwritingPipeline {107 static let margin: CGFloat = 16108 static let maximumPixels: CGFloat = 4096109110 /// The strokes to read: all of them without a region; with one, those whose ink lies mostly inside it (its centre in the111 /// region, or at least half of its box).112 static func strokeIndices(of drawing: PKDrawing, in region: CGRect?) -> [Int] {113 guard let region else { return Array(drawing.strokes.indices) }114 return drawing.strokes.indices.filter { index in115 let box = drawing.strokes[index].renderBounds116 guard !box.isNull, box.intersects(region) else { return false }117 if region.contains(CGPoint(x: box.midX, y: box.midY)) { return true }118 let shared = box.intersection(region), area = max(box.width * box.height, 1)119 return shared.width * shared.height / area >= 0.5120 }121 }122123 static func drawing(_ drawing: PKDrawing, keeping indices: [Int]) -> PKDrawing {124 let wanted = Set(indices)125 return PKDrawing(strokes: drawing.strokes.enumerated().filter { wanted.contains($0.offset) }.map(\.element))126 }127128 static func drawing(_ drawing: PKDrawing, removing indices: [Int]) -> PKDrawing {129 let unwanted = Set(indices)130 return PKDrawing(strokes: drawing.strokes.enumerated().filter { !unwanted.contains($0.offset) }.map(\.element))131 }132133 /// Only the chosen strokes, on white, as in light mode, at the highest scale (up to 4×) that keeps the picture under 4096 px.134 @MainActor135 static func bitmap(of drawing: PKDrawing, indices: [Int]) -> CGImage? {136 let selection = self.drawing(drawing, keeping: indices)137 let bounds = selection.bounds138 guard !selection.strokes.isEmpty, !bounds.isNull, !bounds.isInfinite, bounds.width > 0 || bounds.height > 0 else { return nil }139 let rect = bounds.insetBy(dx: -margin, dy: -margin).integral140 let scale = max(1, min(4, maximumPixels / max(rect.width, rect.height)))141 var ink: UIImage?142 UITraitCollection(userInterfaceStyle: .light).performAsCurrent { ink = selection.image(from: rect, scale: scale) }143 let format = UIGraphicsImageRendererFormat()144 format.scale = scale145 format.opaque = true146 format.preferredRange = .standard147 return UIGraphicsImageRenderer(size: rect.size, format: format).image { context in148 UIColor.white.setFill()149 context.fill(CGRect(origin: .zero, size: rect.size))150 ink?.draw(in: CGRect(origin: .zero, size: rect.size))151 }.cgImage152 }153154 /// Selection → picture → recogniser → proposal. Nil when the region holds no stroke.155 @MainActor156 static func recognize(_ drawing: PKDrawing, region: CGRect?, recognizer: any HandwritingRecognizing) async -> HandwritingProposal? {157 let indices = strokeIndices(of: drawing, in: region)158 guard !indices.isEmpty, let image = bitmap(of: drawing, indices: indices) else { return nil }159 return HandwritingProposal(strokeIndices: indices, result: HandwritingResult(lines: await recognizer.recognize(image)))160 }161}162