SPB Git

Repository Public License

TranscriptionEngine.swift 15.5 KB 293 lines swift

1//2//  ORIX Audio3//  File: Core/Sources/ORIXAudioCore/Speech/TranscriptionEngine.swift4//  Description: On-device transcription of audio files with SpeechAnalyzer / SpeechTranscriber, with an on-device-only SFSpeechRecognizer fallback.5//6//  Author: Simon-Pierre Boucher7//  Contact: contact@spboucher.ai8//9import AVFoundation10import CoreMedia11import Foundation12import Speech1314public enum TranscriptionError: Error, LocalizedError, Sendable {15    case notAuthorized16    case unsupportedLocale(String)17    case modelNotInstalled(String)18    case onDeviceUnavailable(String)19    case failed(String)2021    public var errorDescription: String? {22        switch self {23        case .notAuthorized: "Speech recognition isn't authorised for ORIX Audio. You can allow it in Settings."24        case .unsupportedLocale(let l): "On-device transcription isn't available for \(l) on this device."25        case .modelNotInstalled(let l): "The on-device speech model for \(l) isn't installed. It must be downloaded once (this needs a network connection); audio is never uploaded."26        case .onDeviceUnavailable(let l): "This device can't transcribe \(l) on-device, and ORIX Audio never sends audio to a server."27        case .failed(let m): "Transcription failed: \(m)"28        }29    }30}3132/// Which recogniser can serve a locale on this device, right now.33public enum TranscriptionPath: Sendable, Equatable {34    /// SpeechTranscriber with its model installed.35    case speechTranscriber36    /// SpeechTranscriber supports the locale but its model must be downloaded first.37    case speechTranscriberNeedsDownload38    /// Legacy recogniser, restricted to on-device recognition.39    case legacyOnDevice40    case unavailable4142    public var engineName: String {43        switch self {44        case .speechTranscriber, .speechTranscriberNeedsDownload: "SpeechTranscriber (on-device)"45        case .legacyOnDevice: "SFSpeechRecognizer (on-device only)"46        case .unavailable: "—"47        }48    }49}5051public enum TranscriptionEngine {52    /// Locales the system reports as transcribable on-device, by either path. Nothing is advertised beyond this list.53    public static func supportedLocales() async -> [Locale] {54        var ids = Set<String>()55        var out: [Locale] = []56        if SpeechTranscriber.isAvailable {57            for l in await SpeechTranscriber.supportedLocales where ids.insert(l.identifier(.bcp47)).inserted { out.append(l) }58        }59        for l in SFSpeechRecognizer.supportedLocales() where !ids.contains(l.identifier(.bcp47)) {60            if let r = SFSpeechRecognizer(locale: l), r.supportsOnDeviceRecognition { ids.insert(l.identifier(.bcp47)); out.append(l) }61        }62        return out.sorted { $0.identifier < $1.identifier }63    }6465    public static func path(for locale: Locale) async -> TranscriptionPath {66        if SpeechTranscriber.isAvailable, let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale) {67            let installed = await SpeechTranscriber.installedLocales.contains { $0.identifier(.bcp47) == match.identifier(.bcp47) }68            return installed ? .speechTranscriber : .speechTranscriberNeedsDownload69        }70        if let r = SFSpeechRecognizer(locale: locale), r.supportsOnDeviceRecognition, r.isAvailable { return .legacyOnDevice }71        return .unavailable72    }7374    public static func requestAuthorization() async -> Bool {75        await withCheckedContinuation { c in SFSpeechRecognizer.requestAuthorization { c.resume(returning: $0 == .authorized) } }76    }7778    /// Downloads the on-device model for a locale. This is the only step that touches the network, and it transfers a model, never audio.79    public static func installModel(for locale: Locale, progress: (@Sendable (Double) -> Void)? = nil) async throws {80        guard SpeechTranscriber.isAvailable, let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else {81            throw TranscriptionError.unsupportedLocale(locale.identifier)82        }83        let transcriber = SpeechTranscriber(locale: match, preset: .transcription)84        guard let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else { return }85        let observation = request.progress.observe(\.fractionCompleted) { p, _ in progress?(p.fractionCompleted) }86        defer { observation.invalidate() }87        try await request.downloadAndInstall()88    }8990    /// Transcribes a file entirely on-device. Throws rather than falling back to any server-based recognition.91    /// - Parameter allowModelDownload: when false and the model is missing, throws `.modelNotInstalled`.92    public static func transcribe(url: URL, locale: Locale, allowModelDownload: Bool = false,93                                  progress: (@Sendable (Double) -> Void)? = nil) async throws -> Transcript {94        switch await path(for: locale) {95        case .speechTranscriberNeedsDownload:96            guard allowModelDownload else { throw TranscriptionError.modelNotInstalled(locale.identifier) }97            try await installModel(for: locale) { progress?($0 * 0.1) }98            return try await withSpeechTranscriber(url: url, locale: locale, progress: progress)99        case .speechTranscriber:100            return try await withSpeechTranscriber(url: url, locale: locale, progress: progress)101        case .legacyOnDevice:102            return try await withLegacyRecognizer(url: url, locale: locale, progress: progress)103        case .unavailable:104            throw TranscriptionError.onDeviceUnavailable(locale.identifier)105        }106    }107108    // MARK: SpeechAnalyzer path109110    private static func withSpeechTranscriber(url: URL, locale: Locale, progress: (@Sendable (Double) -> Void)?) async throws -> Transcript {111        guard let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { throw TranscriptionError.unsupportedLocale(locale.identifier) }112        let transcriber = SpeechTranscriber(locale: match, transcriptionOptions: [], reportingOptions: [], attributeOptions: [.audioTimeRange])113        let analyzer = SpeechAnalyzer(modules: [transcriber])114        let file: AVAudioFile115        do { file = try AVAudioFile(forReading: url) } catch { throw AudioIOError.unreadable(error.localizedDescription) }116        let duration = Double(file.length) / file.fileFormat.sampleRate117118        let collector = Task { () throws -> [TranscriptSegment] in119            var segments: [TranscriptSegment] = []120            for try await result in transcriber.results {121                segments += Self.segments(from: result.text, fallbackRange: result.range)122                if duration > 0 { progress?(min(1, result.range.end.seconds / duration)) }123            }124            return segments125        }126        do {127            try await withTaskCancellationHandler {128                if let last = try await analyzer.analyzeSequence(from: file) {129                    try await analyzer.finalizeAndFinish(through: last)130                } else {131                    await analyzer.cancelAndFinishNow()132                }133            } onCancel: {134                Task { await analyzer.cancelAndFinishNow() }135            }136            try Task.checkCancellation()137            let segments = try await collector.value138            progress?(1)139            return Transcript(localeIdentifier: match.identifier(.bcp47), engine: TranscriptionPath.speechTranscriber.engineName,140                              segments: segments.sorted { $0.start < $1.start })141        } catch is CancellationError {142            collector.cancel(); throw CancellationError()143        } catch {144            collector.cancel()145            throw TranscriptionError.failed(error.localizedDescription)146        }147    }148149    /// Splits one recogniser result into readable segments using the per-run audio time ranges.150    static func segments(from text: AttributedString, fallbackRange: CMTimeRange, maxDuration: Double = 14) -> [TranscriptSegment] {151        struct Piece { var text: String; var start: Double; var end: Double }152        var pieces: [Piece] = []153        for run in text.runs {154            let s = String(text[run.range].characters)155            guard !s.isEmpty else { continue }156            if let r = run.audioTimeRange, r.start.isNumeric, r.end.isNumeric {157                pieces.append(Piece(text: s, start: r.start.seconds, end: r.end.seconds))158            } else if var last = pieces.popLast() {159                last.text += s; pieces.append(last)           // untimed punctuation / whitespace joins the previous piece160            } else {161                pieces.append(Piece(text: s, start: fallbackRange.start.seconds, end: fallbackRange.start.seconds))162            }163        }164        let whole = String(text.characters).trimmingCharacters(in: .whitespacesAndNewlines)165        guard !whole.isEmpty else { return [] }166        guard !pieces.isEmpty else { return [TranscriptSegment(start: fallbackRange.start.seconds, end: fallbackRange.end.seconds, text: whole)] }167        var out: [TranscriptSegment] = []168        var current: [Piece] = []169        func flush() {170            guard let first = current.first, let last = current.last else { return }171            let t = current.map(\.text).joined().trimmingCharacters(in: .whitespacesAndNewlines)172            if !t.isEmpty {173                let words = current.compactMap { p -> TranscriptWord? in174                    let w = p.text.trimmingCharacters(in: .whitespacesAndNewlines)175                    return w.isEmpty ? nil : TranscriptWord(text: w, start: p.start, end: p.end)176                }177                out.append(TranscriptSegment(start: first.start, end: max(last.end, first.start), text: t, words: words))178            }179            current = []180        }181        for p in pieces {182            current.append(p)183            let trimmed = p.text.trimmingCharacters(in: .whitespaces)184            let endsSentence = trimmed.last.map { ".!?…".contains($0) } ?? false185            let length = p.end - (current.first?.start ?? p.start)186            if (endsSentence && current.count >= 3) || length > maxDuration { flush() }187        }188        flush()189        return out190    }191192    // MARK: Legacy path (on-device only)193194    private static func withLegacyRecognizer(url: URL, locale: Locale, progress: (@Sendable (Double) -> Void)?) async throws -> Transcript {195        guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition else {196            throw TranscriptionError.onDeviceUnavailable(locale.identifier)197        }198        let request = SFSpeechURLRecognitionRequest(url: url)199        request.requiresOnDeviceRecognition = true          // never falls back to server recognition200        request.shouldReportPartialResults = false201        request.addsPunctuation = true202        final class Box: @unchecked Sendable { var task: SFSpeechRecognitionTask?; var resumed = false }203        let box = Box()204        let words: [TranscriptWord] = try await withTaskCancellationHandler {205            try await withCheckedThrowingContinuation { (c: CheckedContinuation<[TranscriptWord], Error>) in206                box.task = recognizer.recognitionTask(with: request) { result, error in207                    guard !box.resumed else { return }208                    if let error { box.resumed = true; c.resume(throwing: TranscriptionError.failed(error.localizedDescription)); return }209                    guard let result, result.isFinal else { return }210                    box.resumed = true211                    c.resume(returning: result.bestTranscription.segments.map {212                        TranscriptWord(text: $0.substring, start: $0.timestamp, end: $0.timestamp + $0.duration, confidence: Double($0.confidence))213                    })214                }215            }216        } onCancel: { box.task?.cancel() }217        progress?(1)218        return Transcript(localeIdentifier: locale.identifier(.bcp47), engine: TranscriptionPath.legacyOnDevice.engineName, segments: Transcript.segment(words: words))219    }220}221222/// Live (while recording) transcription. Strictly optional: capture never depends on it.223public actor LiveTranscriber {224    public struct Update: Sendable {225        /// Text the recogniser has committed.226        public var finalized: String227        /// Text still subject to change.228        public var volatile: String229    }230231    private var analyzer: SpeechAnalyzer?232    private var continuation: AsyncStream<AnalyzerInput>.Continuation?233    private var converter: AVAudioConverter?234    private var analyzerFormat: AVAudioFormat?235    private var resultsTask: Task<Void, Never>?236    private var finalized = ""237238    public init() {}239240    /// Returns false when live transcription isn't possible for this locale right now (no model installed, unsupported…).241    public func start(locale: Locale, inputFormat: AVAudioFormat, onUpdate: @escaping @Sendable (Update) -> Void) async -> Bool {242        guard SpeechTranscriber.isAvailable, let match = await SpeechTranscriber.supportedLocale(equivalentTo: locale),243              await SpeechTranscriber.installedLocales.contains(where: { $0.identifier(.bcp47) == match.identifier(.bcp47) }) else { return false }244        let transcriber = SpeechTranscriber(locale: match, transcriptionOptions: [], reportingOptions: [.volatileResults], attributeOptions: [])245        guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber], considering: inputFormat) else { return false }246        analyzerFormat = format247        converter = AVAudioConverter(from: inputFormat, to: format)248        let (stream, continuation) = AsyncStream<AnalyzerInput>.makeStream(bufferingPolicy: .bufferingNewest(64))249        self.continuation = continuation250        let analyzer = SpeechAnalyzer(modules: [transcriber])251        self.analyzer = analyzer252        finalized = ""253        do { try await analyzer.start(inputSequence: stream) } catch { return false }254        resultsTask = Task { [weak self] in255            do {256                for try await result in transcriber.results {257                    let text = String(result.text.characters)258                    await self?.handle(text: text, isFinal: result.isFinal, onUpdate: onUpdate)259                }260            } catch {}261        }262        return true263    }264265    private func handle(text: String, isFinal: Bool, onUpdate: @Sendable (Update) -> Void) {266        if isFinal { finalized += text; onUpdate(Update(finalized: finalized, volatile: "")) }267        else { onUpdate(Update(finalized: finalized, volatile: text)) }268    }269270    /// Feeds captured audio. Called from the analysis thread; buffers are dropped (not queued unboundedly) when the recogniser falls behind.271    public func feed(_ buffer: AVAudioPCMBuffer) {272        guard let continuation, let converter, let analyzerFormat else { return }273        let capacity = AVAudioFrameCount(Double(buffer.frameLength) * analyzerFormat.sampleRate / buffer.format.sampleRate) + 64274        guard let out = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return }275        nonisolated(unsafe) var supplied = false276        nonisolated(unsafe) let source = buffer277        var error: NSError?278        converter.convert(to: out, error: &error) { _, status in279            if supplied { status.pointee = .noDataNow; return nil }280            supplied = true; status.pointee = .haveData; return source281        }282        if error == nil, out.frameLength > 0 { continuation.yield(AnalyzerInput(buffer: out)) }283    }284285    public func stop() async -> String {286        continuation?.finish(); continuation = nil287        try? await analyzer?.finalizeAndFinishThroughEndOfInput()288        await resultsTask?.value289        analyzer = nil; resultsTask = nil; converter = nil290        return finalized291    }292}293
Files main
Menu

Theme