SPB Git

Repository Public

Providers.swift 18.3 KB 331 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : SP Lecture Studio9// File             : Packages/LectureTranscription/Sources/Providers.swift10// Description      : Real providers: Apple on-device speech recognition, an11//                    optional user-configured HTTPS endpoint, the project12//                    transcriber and live phrase spotting.13// Module           : Packages14// Repository       : https://git.spboucher.ai/sp-lecture-studio15// Created          : 2026-11-1816// Last Modified    : 2026-11-1817//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : None specified (no license file in this repository)83// Maintainer       : Simon-Pierre Boucher84// Technical Contact: dev@spboucher.ai85//86// =============================================================================87// END OF SP BOUCHER SOURCE HEADER88// =============================================================================8990import Foundation91import AVFoundation92import Speech93import LectureCore94import LectureStorage95import LectureRender9697/// Apple speech recognition, strictly on device: audio never leaves the machine. When the on-device model for the98/// language is not installed the provider reports itself unavailable instead of falling back to Apple's servers.99public struct AppleOnDeviceSpeechProvider: TranscriptionProvider {100    public let id = "apple.speech.ondevice"101    public let sendsAudioOffDevice = false102    public init() {}103104    public static func authorization() -> SFSpeechRecognizerAuthorizationStatus { SFSpeechRecognizer.authorizationStatus() }105    public static func requestAuthorization() async -> Bool {106        await withCheckedContinuation { c in SFSpeechRecognizer.requestAuthorization { c.resume(returning: $0 == .authorized) } }107    }108109    public func availability(locale: Locale) async -> ProviderAvailability {110        guard SFSpeechRecognizer.authorizationStatus() == .authorized else { return .unavailable("permission") }111        guard let r = SFSpeechRecognizer(locale: locale) else { return .unavailable("unsupportedLocale") }112        guard r.supportsOnDeviceRecognition else { return .unavailable("modelNotInstalled") }113        guard r.isAvailable else { return .unavailable("temporarilyUnavailable") }114        return .available115    }116117    public func transcribe(_ chunk: AudioChunk, locale: Locale) async throws -> [TranscriptSegment] {118        guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition else { throw TranscriptionError.unavailable("modelNotInstalled") }119        let request = SFSpeechAudioBufferRecognitionRequest()120        request.requiresOnDeviceRecognition = true121        request.shouldReportPartialResults = false122        request.addsPunctuation = true123        guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: chunk.sampleRate, channels: 1, interleaved: false) else { throw TranscriptionError.failed("format") }124        var pos = 0125        while pos < chunk.samples.count {126            let n = min(16_384, chunk.samples.count - pos)127            guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(n)) else { break }128            buffer.frameLength = AVAudioFrameCount(n)129            chunk.samples.withUnsafeBufferPointer { buffer.floatChannelData![0].update(from: $0.baseAddress!.advanced(by: pos), count: n) }130            request.append(buffer); pos += n131        }132        request.endAudio()133        final class Box: @unchecked Sendable { var task: SFSpeechRecognitionTask?; var done = false }134        let box = Box()135        return try await withTaskCancellationHandler {136            try await withCheckedThrowingContinuation { (cont: CheckedContinuation<[TranscriptSegment], Error>) in137                box.task = recognizer.recognitionTask(with: request) { result, error in138                    guard !box.done else { return }139                    if let result, result.isFinal {140                        box.done = true141                        let words = result.bestTranscription.segments.map {142                            RecognizedWord(text: $0.substring, start: $0.timestamp, duration: $0.duration, confidence: Double($0.confidence))143                        }144                        cont.resume(returning: TranscriptAssembler.group(words, offset: .zero))145                    } else if let error {146                        box.done = true147                        let e = error as NSError148                        // "No speech detected" is a normal outcome for a silent chunk.149                        if e.domain == "kAFAssistantErrorDomain" && (e.code == 1110 || e.code == 203) { cont.resume(returning: []) }150                        else { cont.resume(throwing: TranscriptionError.failed(error.localizedDescription)) }151                    }152                }153            }154        } onCancel: { box.task?.cancel() }155    }156}157158/// Optional provider for a server chosen and configured by the user (for example a Whisper server on their own cluster).159/// It speaks the widely implemented OpenAI-compatible `POST …/audio/transcriptions` multipart API. Nothing is assumed160/// about any particular host: the endpoint URL, model name and token all come from the user's settings.161public struct RemoteTranscriptionProvider: TranscriptionProvider {162    public struct Configuration: Sendable, Hashable, Codable {163        /// Full URL of the transcription endpoint, e.g. https://host.example/v1/audio/transcriptions164        public var endpoint: String165        public var model: String166        public init(endpoint: String = "", model: String = "whisper-1") { self.endpoint = endpoint; self.model = model }167        public var url: URL? { URL(string: endpoint.trimmingCharacters(in: .whitespaces)) }168        /// Audio is only ever sent over TLS (plain HTTP is accepted for the local machine only, for testing a local server).169        public var isSecure: Bool {170            guard let u = url, let scheme = u.scheme?.lowercased(), let host = u.host else { return false }171            return scheme == "https" || (scheme == "http" && ["localhost", "127.0.0.1", "::1"].contains(host))172        }173    }174175    public let id = "remote.openai-compatible"176    public let sendsAudioOffDevice = true177    public let configuration: Configuration178    private let token: String?179    private let session: URLSession180    public var maxAttempts = 3181182    public init(configuration: Configuration, token: String?, session: URLSession = .shared) {183        self.configuration = configuration; self.token = token; self.session = session184    }185186    public func availability(locale: Locale) async -> ProviderAvailability {187        guard configuration.url != nil, !configuration.endpoint.isEmpty else { return .unavailable("notConfigured") }188        guard configuration.isSecure else { return .unavailable("insecureEndpoint") }189        return .available190    }191192    public func transcribe(_ chunk: AudioChunk, locale: Locale) async throws -> [TranscriptSegment] {193        guard let url = configuration.url, configuration.isSecure else { throw TranscriptionError.unavailable("insecureEndpoint") }194        let wav = WAV.encode16k(chunk.samples, sampleRate: chunk.sampleRate)195        let boundary = "slecture-\(UUID().uuidString)"196        var body = Data()197        func field(_ name: String, _ value: String) { body.append(Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"\(name)\"\r\n\r\n\(value)\r\n".utf8)) }198        field("model", configuration.model); field("response_format", "verbose_json")199        if let lang = locale.language.languageCode?.identifier { field("language", lang) }200        body.append(Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\nContent-Type: audio/wav\r\n\r\n".utf8))201        body.append(wav); body.append(Data("\r\n--\(boundary)--\r\n".utf8))202        var request = URLRequest(url: url)203        request.httpMethod = "POST"; request.timeoutInterval = 300204        request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")205        if let token, !token.isEmpty { request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") }206207        var lastError: Error = TranscriptionError.failed("no attempt")208        for attempt in 0..<maxAttempts {209            try Task.checkCancellation()210            do {211                let (data, response) = try await session.upload(for: request, from: body)212                let code = (response as? HTTPURLResponse)?.statusCode ?? 0213                if (200..<300).contains(code) { return try Self.parse(data, chunkDuration: chunk.range.duration.seconds) }214                let message = String(decoding: data.prefix(300), as: UTF8.self)215                // Client errors will not improve by retrying (bad token, bad request).216                if (400..<500).contains(code) && code != 408 && code != 429 { throw TranscriptionError.server(code, message) }217                lastError = TranscriptionError.server(code, message)218            } catch let e as TranscriptionError { if case .server(let c, _) = e, (400..<500).contains(c), c != 408, c != 429 { throw e }; lastError = e }219            catch is CancellationError { throw TranscriptionError.cancelled }220            catch { lastError = TranscriptionError.failed(error.localizedDescription) }221            if attempt < maxAttempts - 1 { try? await Task.sleep(nanoseconds: UInt64(pow(2.0, Double(attempt)) * 0.5 * 1e9)) }222        }223        throw lastError224    }225226    static func parse(_ data: Data, chunkDuration: Double) throws -> [TranscriptSegment] {227        struct Response: Decodable { struct Seg: Decodable { var start: Double; var end: Double; var text: String }; var text: String?; var segments: [Seg]? }228        guard let r = try? JSONDecoder().decode(Response.self, from: data) else { throw TranscriptionError.failed("unexpected response format") }229        if let segs = r.segments, !segs.isEmpty {230            return segs.map { TranscriptSegment(range: TimeRange(seconds: max(0, $0.start), min(chunkDuration, max($0.end, $0.start + 0.1))), text: $0.text.trimmingCharacters(in: .whitespaces)) }231        }232        let text = (r.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)233        return text.isEmpty ? [] : [TranscriptSegment(range: TimeRange(seconds: 0, chunkDuration), text: text)]234    }235}236237enum WAV {238    /// 16 kHz, 16-bit mono WAV (speech models work at 16 kHz; a third of the upload size of 48 kHz).239    static func encode16k(_ samples: [Float], sampleRate: Double) -> Data {240        let ratio = sampleRate / 16_000241        let n = Int(Double(samples.count) / ratio)242        var pcm = Data(capacity: n * 2)243        let window = max(1, Int(ratio.rounded()))244        for i in 0..<n {245            let start = Int(Double(i) * ratio)246            var acc: Float = 0; var count = 0247            for j in start..<min(samples.count, start + window) { acc += samples[j]; count += 1 }      // box filter before decimation248            let v = Int16(max(-1, min(1, count > 0 ? acc / Float(count) : 0)) * 32767)249            withUnsafeBytes(of: v.littleEndian) { pcm.append(contentsOf: $0) }250        }251        var d = Data()252        func u32(_ v: UInt32) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } }253        func u16(_ v: UInt16) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } }254        d.append(Data("RIFF".utf8)); u32(UInt32(36 + pcm.count)); d.append(Data("WAVEfmt ".utf8)); u32(16); u16(1); u16(1); u32(16_000); u32(32_000); u16(2); u16(16)255        d.append(Data("data".utf8)); u32(UInt32(pcm.count)); d.append(pcm)256        return d257    }258}259260/// Transcribes the whole recording of a project, chunk by chunk, on the source timeline.261public enum ProjectTranscriber {262    public static func transcribe(package: ProjectPackage, manifest: ProjectManifest, provider: any TranscriptionProvider, locale: Locale,263                                  progress: @escaping @Sendable (Double) -> Void = { _ in }) async throws -> Transcript {264        let availability = await provider.availability(locale: locale)265        if case .unavailable(let reason) = availability { throw TranscriptionError.unavailable(reason) }266        let reader = TimelineAudioReader(package: package, manifest: manifest, sampleRate: 16_000)267        guard reader.hasAudio, manifest.duration > .zero else { throw TranscriptionError.unavailable("noAudio") }268        let envelope = package.loadEnvelope()269        let plan = ChunkPlanner.plan(duration: manifest.duration, envelope: envelope)270        var results: [(range: TimeRange, segments: [TranscriptSegment])] = []271        for (i, range) in plan.enumerated() {272            if Task.isCancelled { throw TranscriptionError.cancelled }273            let start = range.start.frames(atSampleRate: 16_000), count = Int(range.duration.frames(atSampleRate: 16_000))274            let samples = reader.read(start: start, count: count)275            // Skip chunks that are digital silence: nothing to recognize.276            if (samples.map { abs($0) }.max() ?? 0) > 0.003 {277                let segs = try await provider.transcribe(AudioChunk(range: range, samples: samples, sampleRate: 16_000), locale: locale)278                results.append((range, segs))279            }280            progress(Double(i + 1) / Double(plan.count))281        }282        return TranscriptAssembler.assemble(chunks: results, locale: locale.identifier, provider: provider.id)283    }284}285286/// Live, on-device phrase spotting during a recording (optional voice commands). Audio buffers come from the capture path.287public final class LivePhraseSpotter: @unchecked Sendable {288    private let recognizer: SFSpeechRecognizer289    private var request: SFSpeechAudioBufferRecognitionRequest?290    private var task: SFSpeechRecognitionTask?291    private var matchers: [(PhraseMatcher, () -> Void)] = []292    private let lock = NSLock()293    private var startedAt = Date()294    public private(set) var isRunning = false295296    /// Returns nil when on-device recognition is not available for the locale (the feature is then simply off).297    public init?(locale: Locale) {298        guard SFSpeechRecognizer.authorizationStatus() == .authorized, let r = SFSpeechRecognizer(locale: locale), r.supportsOnDeviceRecognition else { return nil }299        recognizer = r300    }301302    public func watch(phrase: String, action: @escaping () -> Void) { lock.lock(); matchers.append((PhraseMatcher(phrase: phrase), action)); lock.unlock() }303304    public func start() { lock.lock(); isRunning = true; restartLocked(); lock.unlock() }305    public func stop() { lock.lock(); isRunning = false; task?.cancel(); request?.endAudio(); task = nil; request = nil; lock.unlock() }306307    private func restartLocked() {308        task?.cancel(); request?.endAudio()309        let req = SFSpeechAudioBufferRecognitionRequest()310        req.requiresOnDeviceRecognition = true; req.shouldReportPartialResults = true311        request = req; startedAt = Date()312        for i in matchers.indices { matchers[i].0.reset() }313        task = recognizer.recognitionTask(with: req) { [weak self] result, _ in314            guard let self, let text = result?.bestTranscription.formattedString else { return }315            self.lock.lock()316            var fired: [() -> Void] = []317            for i in self.matchers.indices where self.matchers[i].0.feed(text) { fired.append(self.matchers[i].1) }318            self.lock.unlock()319            fired.forEach { action in DispatchQueue.main.async { action() } }320        }321    }322323    public func append(_ buffer: AVAudioPCMBuffer) {324        lock.lock(); defer { lock.unlock() }325        guard isRunning else { return }326        // Recognition tasks are kept short so that latency and memory stay bounded over a long lecture.327        if Date().timeIntervalSince(startedAt) > 50 { restartLocked() }328        request?.append(buffer)329    }330}331
Files main
Menu

Theme