Repository Public License
1//2// ORIX Audio3// File: Core/Sources/ORIXAudioCore/Speech/TranscriptionEngine.swift4// Description: On-device transcription of audio files with SpeechAnalyzer / SpeechTranscriber, with an on-device-only SFSpeechRecognizer fallback.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8//9import AVFoundation10import CoreMedia11import Foundation12import Speech1314public enum TranscriptionError: Error, LocalizedError, Sendable {15 case notAuthorized16 case unsupportedLocale(String)17 case modelNotInstalled(String)18 case onDeviceUnavailable(String)19 case failed(String)2021 public var errorDescription: String? {22 switch self {23 case .notAuthorized: "Speech recognition isn't authorised for ORIX Audio. You can allow it in Settings."24 case .unsupportedLocale(let l): "On-device transcription isn't available for \(l) on this device."25 case .modelNotInstalled(let l): "The on-device speech model for \(l) isn't installed. It must be downloaded once (this needs a network connection); audio is never uploaded."26 case .onDeviceUnavailable(let l): "This device can't transcribe \(l) on-device, and ORIX Audio never sends audio to a server."27 case .failed(let m): "Transcription failed: \(m)"28 }29 }30}3132/// Which recogniser can serve a locale on this device, right now.33public enum TranscriptionPath: Sendable, Equatable {34 /// SpeechTranscriber with its model installed.35 case speechTranscriber36 /// SpeechTranscriber supports the locale but its model must be downloaded first.37 case speechTranscriberNeedsDownload38 /// Legacy recogniser, restricted to on-device recognition.39 case legacyOnDevice40 case unavailable4142 public var engineName: String {43 switch self {44 case .speechTranscriber, .speechTranscriberNeedsDownload: "SpeechTranscriber (on-device)"45 case .legacyOnDevice: "SFSpeechRecognizer (on-device only)"46 case .unavailable: "—"47 }48 }49}5051public enum TranscriptionEngine {52 /// Locales the system reports as transcribable on-device, by either path. Nothing is advertised beyond this list.53 public static func supportedLocales() async -> [Locale] {54 var ids = Set<String>()55 var out: [Locale] = []56 if SpeechTranscriber.isAvailable {57 for l in await SpeechTranscriber.supportedLocales where ids.insert(l.identifier(.bcp47)).inserted { out.append(l) }58 }59 for l in SFSpeechRecognizer.supportedLocales() where !ids.contains(l.identifier(.bcp47)) {60 if let r = SFSpeechRecognizer(locale: l), r.supportsOnDeviceRecognition { ids.insert(l.identifier(.bcp47)); out.append(l) }61 }62 return out.sorted { $0.identifier < $1.identifier }63 }6465 public static func path(for locale: Locale) async -> TranscriptionPath {66 if SpeechTranscriber.isAvailable, let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale) {67 let installed = await SpeechTranscriber.installedLocales.contains { $0.identifier(.bcp47) == match.identifier(.bcp47) }68 return installed ? .speechTranscriber : .speechTranscriberNeedsDownload69 }70 if let r = SFSpeechRecognizer(locale: locale), r.supportsOnDeviceRecognition, r.isAvailable { return .legacyOnDevice }71 return .unavailable72 }7374 public static func requestAuthorization() async -> Bool {75 await withCheckedContinuation { c in SFSpeechRecognizer.requestAuthorization { c.resume(returning: $0 == .authorized) } }76 }7778 /// Downloads the on-device model for a locale. This is the only step that touches the network, and it transfers a model, never audio.79 public static func installModel(for locale: Locale, progress: (@Sendable (Double) -> Void)? = nil) async throws {80 guard SpeechTranscriber.isAvailable, let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else {81 throw TranscriptionError.unsupportedLocale(locale.identifier)82 }83 let transcriber = SpeechTranscriber(locale: match, preset: .transcription)84 guard let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else { return }85 let observation = request.progress.observe(\.fractionCompleted) { p, _ in progress?(p.fractionCompleted) }86 defer { observation.invalidate() }87 try await request.downloadAndInstall()88 }8990 /// Transcribes a file entirely on-device. Throws rather than falling back to any server-based recognition.91 /// - Parameter allowModelDownload: when false and the model is missing, throws `.modelNotInstalled`.92 public static func transcribe(url: URL, locale: Locale, allowModelDownload: Bool = false,93 progress: (@Sendable (Double) -> Void)? = nil) async throws -> Transcript {94 switch await path(for: locale) {95 case .speechTranscriberNeedsDownload:96 guard allowModelDownload else { throw TranscriptionError.modelNotInstalled(locale.identifier) }97 try await installModel(for: locale) { progress?($0 * 0.1) }98 return try await withSpeechTranscriber(url: url, locale: locale, progress: progress)99 case .speechTranscriber:100 return try await withSpeechTranscriber(url: url, locale: locale, progress: progress)101 case .legacyOnDevice:102 return try await withLegacyRecognizer(url: url, locale: locale, progress: progress)103 case .unavailable:104 throw TranscriptionError.onDeviceUnavailable(locale.identifier)105 }106 }107108 // MARK: SpeechAnalyzer path109110 private static func withSpeechTranscriber(url: URL, locale: Locale, progress: (@Sendable (Double) -> Void)?) async throws -> Transcript {111 guard let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { throw TranscriptionError.unsupportedLocale(locale.identifier) }112 let transcriber = SpeechTranscriber(locale: match, transcriptionOptions: [], reportingOptions: [], attributeOptions: [.audioTimeRange])113 let analyzer = SpeechAnalyzer(modules: [transcriber])114 let file: AVAudioFile115 do { file = try AVAudioFile(forReading: url) } catch { throw AudioIOError.unreadable(error.localizedDescription) }116 let duration = Double(file.length) / file.fileFormat.sampleRate117118 let collector = Task { () throws -> [TranscriptSegment] in119 var segments: [TranscriptSegment] = []120 for try await result in transcriber.results {121 segments += Self.segments(from: result.text, fallbackRange: result.range)122 if duration > 0 { progress?(min(1, result.range.end.seconds / duration)) }123 }124 return segments125 }126 do {127 try await withTaskCancellationHandler {128 if let last = try await analyzer.analyzeSequence(from: file) {129 try await analyzer.finalizeAndFinish(through: last)130 } else {131 await analyzer.cancelAndFinishNow()132 }133 } onCancel: {134 Task { await analyzer.cancelAndFinishNow() }135 }136 try Task.checkCancellation()137 let segments = try await collector.value138 progress?(1)139 return Transcript(localeIdentifier: match.identifier(.bcp47), engine: TranscriptionPath.speechTranscriber.engineName,140 segments: segments.sorted { $0.start < $1.start })141 } catch is CancellationError {142 collector.cancel(); throw CancellationError()143 } catch {144 collector.cancel()145 throw TranscriptionError.failed(error.localizedDescription)146 }147 }148149 /// Splits one recogniser result into readable segments using the per-run audio time ranges.150 static func segments(from text: AttributedString, fallbackRange: CMTimeRange, maxDuration: Double = 14) -> [TranscriptSegment] {151 struct Piece { var text: String; var start: Double; var end: Double }152 var pieces: [Piece] = []153 for run in text.runs {154 let s = String(text[run.range].characters)155 guard !s.isEmpty else { continue }156 if let r = run.audioTimeRange, r.start.isNumeric, r.end.isNumeric {157 pieces.append(Piece(text: s, start: r.start.seconds, end: r.end.seconds))158 } else if var last = pieces.popLast() {159 last.text += s; pieces.append(last) // untimed punctuation / whitespace joins the previous piece160 } else {161 pieces.append(Piece(text: s, start: fallbackRange.start.seconds, end: fallbackRange.start.seconds))162 }163 }164 let whole = String(text.characters).trimmingCharacters(in: .whitespacesAndNewlines)165 guard !whole.isEmpty else { return [] }166 guard !pieces.isEmpty else { return [TranscriptSegment(start: fallbackRange.start.seconds, end: fallbackRange.end.seconds, text: whole)] }167 var out: [TranscriptSegment] = []168 var current: [Piece] = []169 func flush() {170 guard let first = current.first, let last = current.last else { return }171 let t = current.map(\.text).joined().trimmingCharacters(in: .whitespacesAndNewlines)172 if !t.isEmpty {173 let words = current.compactMap { p -> TranscriptWord? in174 let w = p.text.trimmingCharacters(in: .whitespacesAndNewlines)175 return w.isEmpty ? nil : TranscriptWord(text: w, start: p.start, end: p.end)176 }177 out.append(TranscriptSegment(start: first.start, end: max(last.end, first.start), text: t, words: words))178 }179 current = []180 }181 for p in pieces {182 current.append(p)183 let trimmed = p.text.trimmingCharacters(in: .whitespaces)184 let endsSentence = trimmed.last.map { ".!?…".contains($0) } ?? false185 let length = p.end - (current.first?.start ?? p.start)186 if (endsSentence && current.count >= 3) || length > maxDuration { flush() }187 }188 flush()189 return out190 }191192 // MARK: Legacy path (on-device only)193194 private static func withLegacyRecognizer(url: URL, locale: Locale, progress: (@Sendable (Double) -> Void)?) async throws -> Transcript {195 guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition else {196 throw TranscriptionError.onDeviceUnavailable(locale.identifier)197 }198 let request = SFSpeechURLRecognitionRequest(url: url)199 request.requiresOnDeviceRecognition = true // never falls back to server recognition200 request.shouldReportPartialResults = false201 request.addsPunctuation = true202 final class Box: @unchecked Sendable { var task: SFSpeechRecognitionTask?; var resumed = false }203 let box = Box()204 let words: [TranscriptWord] = try await withTaskCancellationHandler {205 try await withCheckedThrowingContinuation { (c: CheckedContinuation<[TranscriptWord], Error>) in206 box.task = recognizer.recognitionTask(with: request) { result, error in207 guard !box.resumed else { return }208 if let error { box.resumed = true; c.resume(throwing: TranscriptionError.failed(error.localizedDescription)); return }209 guard let result, result.isFinal else { return }210 box.resumed = true211 c.resume(returning: result.bestTranscription.segments.map {212 TranscriptWord(text: $0.substring, start: $0.timestamp, end: $0.timestamp + $0.duration, confidence: Double($0.confidence))213 })214 }215 }216 } onCancel: { box.task?.cancel() }217 progress?(1)218 return Transcript(localeIdentifier: locale.identifier(.bcp47), engine: TranscriptionPath.legacyOnDevice.engineName, segments: Transcript.segment(words: words))219 }220}221222/// Live (while recording) transcription. Strictly optional: capture never depends on it.223public actor LiveTranscriber {224 public struct Update: Sendable {225 /// Text the recogniser has committed.226 public var finalized: String227 /// Text still subject to change.228 public var volatile: String229 }230231 private var analyzer: SpeechAnalyzer?232 private var continuation: AsyncStream<AnalyzerInput>.Continuation?233 private var converter: AVAudioConverter?234 private var analyzerFormat: AVAudioFormat?235 private var resultsTask: Task<Void, Never>?236 private var finalized = ""237238 public init() {}239240 /// Returns false when live transcription isn't possible for this locale right now (no model installed, unsupported…).241 public func start(locale: Locale, inputFormat: AVAudioFormat, onUpdate: @escaping @Sendable (Update) -> Void) async -> Bool {242 guard SpeechTranscriber.isAvailable, let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale),243 await SpeechTranscriber.installedLocales.contains(where: { $0.identifier(.bcp47) == match.identifier(.bcp47) }) else { return false }244 let transcriber = SpeechTranscriber(locale: match, transcriptionOptions: [], reportingOptions: [.volatileResults], attributeOptions: [])245 guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber], considering: inputFormat) else { return false }246 analyzerFormat = format247 converter = AVAudioConverter(from: inputFormat, to: format)248 let (stream, continuation) = AsyncStream<AnalyzerInput>.makeStream(bufferingPolicy: .bufferingNewest(64))249 self.continuation = continuation250 let analyzer = SpeechAnalyzer(modules: [transcriber])251 self.analyzer = analyzer252 finalized = ""253 do { try await analyzer.start(inputSequence: stream) } catch { return false }254 resultsTask = Task { [weak self] in255 do {256 for try await result in transcriber.results {257 let text = String(result.text.characters)258 await self?.handle(text: text, isFinal: result.isFinal, onUpdate: onUpdate)259 }260 } catch {}261 }262 return true263 }264265 private func handle(text: String, isFinal: Bool, onUpdate: @Sendable (Update) -> Void) {266 if isFinal { finalized += text; onUpdate(Update(finalized: finalized, volatile: "")) }267 else { onUpdate(Update(finalized: finalized, volatile: text)) }268 }269270 /// Feeds captured audio. Called from the analysis thread; buffers are dropped (not queued unboundedly) when the recogniser falls behind.271 public func feed(_ buffer: AVAudioPCMBuffer) {272 guard let continuation, let converter, let analyzerFormat else { return }273 let capacity = AVAudioFrameCount(Double(buffer.frameLength) * analyzerFormat.sampleRate / buffer.format.sampleRate) + 64274 guard let out = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return }275 nonisolated(unsafe) var supplied = false276 nonisolated(unsafe) let source = buffer277 var error: NSError?278 converter.convert(to: out, error: &error) { _, status in279 if supplied { status.pointee = .noDataNow; return nil }280 supplied = true; status.pointee = .haveData; return source281 }282 if error == nil, out.frameLength > 0 { continuation.yield(AnalyzerInput(buffer: out)) }283 }284285 public func stop() async -> String {286 continuation?.finish(); continuation = nil287 try? await analyzer?.finalizeAndFinishThroughEndOfInput()288 await resultsTask?.value289 analyzer = nil; resultsTask = nil; converter = nil290 return finalized291 }292}293