Repository Public
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : SP Lecture Studio9// File : Packages/LectureTranscription/Sources/Providers.swift10// Description : Real providers: Apple on-device speech recognition, an11// optional user-configured HTTPS endpoint, the project12// transcriber and live phrase spotting.13// Module : Packages14// Repository : https://git.spboucher.ai/sp-lecture-studio15// Created : 2026-11-1816// Last Modified : 2026-11-1817//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author : Simon-Pierre Boucher23// Position : Professor of Finance24// Institution : Université du Québec en Outaouais (UQO)25// Department : Département des sciences administratives26// Organization : SP Boucher27//28// Website : https://spboucher.ai29// Git : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General : info@spboucher.ai36// Professional : contact@spboucher.ai37// Direct : simon@spboucher.ai38// Research : research@spboucher.ai39// Development : dev@spboucher.ai40// Support : support@spboucher.ai41// Security/Admin : admin@spboucher.ai42// Notifications : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification : Public69// Distribution : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License : None specified (no license file in this repository)83// Maintainer : Simon-Pierre Boucher84// Technical Contact: dev@spboucher.ai85//86// =============================================================================87// END OF SP BOUCHER SOURCE HEADER88// =============================================================================8990import Foundation91import AVFoundation92import Speech93import LectureCore94import LectureStorage95import LectureRender9697/// Apple speech recognition, strictly on device: audio never leaves the machine. When the on-device model for the98/// language is not installed the provider reports itself unavailable instead of falling back to Apple's servers.99public struct AppleOnDeviceSpeechProvider: TranscriptionProvider {100 public let id = "apple.speech.ondevice"101 public let sendsAudioOffDevice = false102 public init() {}103104 public static func authorization() -> SFSpeechRecognizerAuthorizationStatus { SFSpeechRecognizer.authorizationStatus() }105 public static func requestAuthorization() async -> Bool {106 await withCheckedContinuation { c in SFSpeechRecognizer.requestAuthorization { c.resume(returning: $0 == .authorized) } }107 }108109 public func availability(locale: Locale) async -> ProviderAvailability {110 guard SFSpeechRecognizer.authorizationStatus() == .authorized else { return .unavailable("permission") }111 guard let r = SFSpeechRecognizer(locale: locale) else { return .unavailable("unsupportedLocale") }112 guard r.supportsOnDeviceRecognition else { return .unavailable("modelNotInstalled") }113 guard r.isAvailable else { return .unavailable("temporarilyUnavailable") }114 return .available115 }116117 public func transcribe(_ chunk: AudioChunk, locale: Locale) async throws -> [TranscriptSegment] {118 guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition else { throw TranscriptionError.unavailable("modelNotInstalled") }119 let request = SFSpeechAudioBufferRecognitionRequest()120 request.requiresOnDeviceRecognition = true121 request.shouldReportPartialResults = false122 request.addsPunctuation = true123 guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: chunk.sampleRate, channels: 1, interleaved: false) else { throw TranscriptionError.failed("format") }124 var pos = 0125 while pos < chunk.samples.count {126 let n = min(16_384, chunk.samples.count - pos)127 guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(n)) else { break }128 buffer.frameLength = AVAudioFrameCount(n)129 chunk.samples.withUnsafeBufferPointer { buffer.floatChannelData![0].update(from: $0.baseAddress!.advanced(by: pos), count: n) }130 request.append(buffer); pos += n131 }132 request.endAudio()133 final class Box: @unchecked Sendable { var task: SFSpeechRecognitionTask?; var done = false }134 let box = Box()135 return try await withTaskCancellationHandler {136 try await withCheckedThrowingContinuation { (cont: CheckedContinuation<[TranscriptSegment], Error>) in137 box.task = recognizer.recognitionTask(with: request) { result, error in138 guard !box.done else { return }139 if let result, result.isFinal {140 box.done = true141 let words = result.bestTranscription.segments.map {142 RecognizedWord(text: $0.substring, start: $0.timestamp, duration: $0.duration, confidence: Double($0.confidence))143 }144 cont.resume(returning: TranscriptAssembler.group(words, offset: .zero))145 } else if let error {146 box.done = true147 let e = error as NSError148 // "No speech detected" is a normal outcome for a silent chunk.149 if e.domain == "kAFAssistantErrorDomain" && (e.code == 1110 || e.code == 203) { cont.resume(returning: []) }150 else { cont.resume(throwing: TranscriptionError.failed(error.localizedDescription)) }151 }152 }153 }154 } onCancel: { box.task?.cancel() }155 }156}157158/// Optional provider for a server chosen and configured by the user (for example a Whisper server on their own cluster).159/// It speaks the widely implemented OpenAI-compatible `POST …/audio/transcriptions` multipart API. Nothing is assumed160/// about any particular host: the endpoint URL, model name and token all come from the user's settings.161public struct RemoteTranscriptionProvider: TranscriptionProvider {162 public struct Configuration: Sendable, Hashable, Codable {163 /// Full URL of the transcription endpoint, e.g. https://host.example/v1/audio/transcriptions164 public var endpoint: String165 public var model: String166 public init(endpoint: String = "", model: String = "whisper-1") { self.endpoint = endpoint; self.model = model }167 public var url: URL? { URL(string: endpoint.trimmingCharacters(in: .whitespaces)) }168 /// Audio is only ever sent over TLS (plain HTTP is accepted for the local machine only, for testing a local server).169 public var isSecure: Bool {170 guard let u = url, let scheme = u.scheme?.lowercased(), let host = u.host else { return false }171 return scheme == "https" || (scheme == "http" && ["localhost", "127.0.0.1", "::1"].contains(host))172 }173 }174175 public let id = "remote.openai-compatible"176 public let sendsAudioOffDevice = true177 public let configuration: Configuration178 private let token: String?179 private let session: URLSession180 public var maxAttempts = 3181182 public init(configuration: Configuration, token: String?, session: URLSession = .shared) {183 self.configuration = configuration; self.token = token; self.session = session184 }185186 public func availability(locale: Locale) async -> ProviderAvailability {187 guard configuration.url != nil, !configuration.endpoint.isEmpty else { return .unavailable("notConfigured") }188 guard configuration.isSecure else { return .unavailable("insecureEndpoint") }189 return .available190 }191192 public func transcribe(_ chunk: AudioChunk, locale: Locale) async throws -> [TranscriptSegment] {193 guard let url = configuration.url, configuration.isSecure else { throw TranscriptionError.unavailable("insecureEndpoint") }194 let wav = WAV.encode16k(chunk.samples, sampleRate: chunk.sampleRate)195 let boundary = "slecture-\(UUID().uuidString)"196 var body = Data()197 func field(_ name: String, _ value: String) { body.append(Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"\(name)\"\r\n\r\n\(value)\r\n".utf8)) }198 field("model", configuration.model); field("response_format", "verbose_json")199 if let lang = locale.language.languageCode?.identifier { field("language", lang) }200 body.append(Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"file\"; filename=\"chunk.wav\"\r\nContent-Type: audio/wav\r\n\r\n".utf8))201 body.append(wav); body.append(Data("\r\n--\(boundary)--\r\n".utf8))202 var request = URLRequest(url: url)203 request.httpMethod = "POST"; request.timeoutInterval = 300204 request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")205 if let token, !token.isEmpty { request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") }206207 var lastError: Error = TranscriptionError.failed("no attempt")208 for attempt in 0..<maxAttempts {209 try Task.checkCancellation()210 do {211 let (data, response) = try await session.upload(for: request, from: body)212 let code = (response as? HTTPURLResponse)?.statusCode ?? 0213 if (200..<300).contains(code) { return try Self.parse(data, chunkDuration: chunk.range.duration.seconds) }214 let message = String(decoding: data.prefix(300), as: UTF8.self)215 // Client errors will not improve by retrying (bad token, bad request).216 if (400..<500).contains(code) && code != 408 && code != 429 { throw TranscriptionError.server(code, message) }217 lastError = TranscriptionError.server(code, message)218 } catch let e as TranscriptionError { if case .server(let c, _) = e, (400..<500).contains(c), c != 408, c != 429 { throw e }; lastError = e }219 catch is CancellationError { throw TranscriptionError.cancelled }220 catch { lastError = TranscriptionError.failed(error.localizedDescription) }221 if attempt < maxAttempts - 1 { try? await Task.sleep(nanoseconds: UInt64(pow(2.0, Double(attempt)) * 0.5 * 1e9)) }222 }223 throw lastError224 }225226 static func parse(_ data: Data, chunkDuration: Double) throws -> [TranscriptSegment] {227 struct Response: Decodable { struct Seg: Decodable { var start: Double; var end: Double; var text: String }; var text: String?; var segments: [Seg]? }228 guard let r = try? JSONDecoder().decode(Response.self, from: data) else { throw TranscriptionError.failed("unexpected response format") }229 if let segs = r.segments, !segs.isEmpty {230 return segs.map { TranscriptSegment(range: TimeRange(seconds: max(0, $0.start), min(chunkDuration, max($0.end, $0.start + 0.1))), text: $0.text.trimmingCharacters(in: .whitespaces)) }231 }232 let text = (r.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)233 return text.isEmpty ? [] : [TranscriptSegment(range: TimeRange(seconds: 0, chunkDuration), text: text)]234 }235}236237enum WAV {238 /// 16 kHz, 16-bit mono WAV (speech models work at 16 kHz; a third of the upload size of 48 kHz).239 static func encode16k(_ samples: [Float], sampleRate: Double) -> Data {240 let ratio = sampleRate / 16_000241 let n = Int(Double(samples.count) / ratio)242 var pcm = Data(capacity: n * 2)243 let window = max(1, Int(ratio.rounded()))244 for i in 0..<n {245 let start = Int(Double(i) * ratio)246 var acc: Float = 0; var count = 0247 for j in start..<min(samples.count, start + window) { acc += samples[j]; count += 1 } // box filter before decimation248 let v = Int16(max(-1, min(1, count > 0 ? acc / Float(count) : 0)) * 32767)249 withUnsafeBytes(of: v.littleEndian) { pcm.append(contentsOf: $0) }250 }251 var d = Data()252 func u32(_ v: UInt32) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } }253 func u16(_ v: UInt16) { withUnsafeBytes(of: v.littleEndian) { d.append(contentsOf: $0) } }254 d.append(Data("RIFF".utf8)); u32(UInt32(36 + pcm.count)); d.append(Data("WAVEfmt ".utf8)); u32(16); u16(1); u16(1); u32(16_000); u32(32_000); u16(2); u16(16)255 d.append(Data("data".utf8)); u32(UInt32(pcm.count)); d.append(pcm)256 return d257 }258}259260/// Transcribes the whole recording of a project, chunk by chunk, on the source timeline.261public enum ProjectTranscriber {262 public static func transcribe(package: ProjectPackage, manifest: ProjectManifest, provider: any TranscriptionProvider, locale: Locale,263 progress: @escaping @Sendable (Double) -> Void = { _ in }) async throws -> Transcript {264 let availability = await provider.availability(locale: locale)265 if case .unavailable(let reason) = availability { throw TranscriptionError.unavailable(reason) }266 let reader = TimelineAudioReader(package: package, manifest: manifest, sampleRate: 16_000)267 guard reader.hasAudio, manifest.duration > .zero else { throw TranscriptionError.unavailable("noAudio") }268 let envelope = package.loadEnvelope()269 let plan = ChunkPlanner.plan(duration: manifest.duration, envelope: envelope)270 var results: [(range: TimeRange, segments: [TranscriptSegment])] = []271 for (i, range) in plan.enumerated() {272 if Task.isCancelled { throw TranscriptionError.cancelled }273 let start = range.start.frames(atSampleRate: 16_000), count = Int(range.duration.frames(atSampleRate: 16_000))274 let samples = reader.read(start: start, count: count)275 // Skip chunks that are digital silence: nothing to recognize.276 if (samples.map { abs($0) }.max() ?? 0) > 0.003 {277 let segs = try await provider.transcribe(AudioChunk(range: range, samples: samples, sampleRate: 16_000), locale: locale)278 results.append((range, segs))279 }280 progress(Double(i + 1) / Double(plan.count))281 }282 return TranscriptAssembler.assemble(chunks: results, locale: locale.identifier, provider: provider.id)283 }284}285286/// Live, on-device phrase spotting during a recording (optional voice commands). Audio buffers come from the capture path.287public final class LivePhraseSpotter: @unchecked Sendable {288 private let recognizer: SFSpeechRecognizer289 private var request: SFSpeechAudioBufferRecognitionRequest?290 private var task: SFSpeechRecognitionTask?291 private var matchers: [(PhraseMatcher, () -> Void)] = []292 private let lock = NSLock()293 private var startedAt = Date()294 public private(set) var isRunning = false295296 /// Returns nil when on-device recognition is not available for the locale (the feature is then simply off).297 public init?(locale: Locale) {298 guard SFSpeechRecognizer.authorizationStatus() == .authorized, let r = SFSpeechRecognizer(locale: locale), r.supportsOnDeviceRecognition else { return nil }299 recognizer = r300 }301302 public func watch(phrase: String, action: @escaping () -> Void) { lock.lock(); matchers.append((PhraseMatcher(phrase: phrase), action)); lock.unlock() }303304 public func start() { lock.lock(); isRunning = true; restartLocked(); lock.unlock() }305 public func stop() { lock.lock(); isRunning = false; task?.cancel(); request?.endAudio(); task = nil; request = nil; lock.unlock() }306307 private func restartLocked() {308 task?.cancel(); request?.endAudio()309 let req = SFSpeechAudioBufferRecognitionRequest()310 req.requiresOnDeviceRecognition = true; req.shouldReportPartialResults = true311 request = req; startedAt = Date()312 for i in matchers.indices { matchers[i].0.reset() }313 task = recognizer.recognitionTask(with: req) { [weak self] result, _ in314 guard let self, let text = result?.bestTranscription.formattedString else { return }315 self.lock.lock()316 var fired: [() -> Void] = []317 for i in self.matchers.indices where self.matchers[i].0.feed(text) { fired.append(self.matchers[i].1) }318 self.lock.unlock()319 fired.forEach { action in DispatchQueue.main.async { action() } }320 }321 }322323 public func append(_ buffer: AVAudioPCMBuffer) {324 lock.lock(); defer { lock.unlock() }325 guard isRunning else { return }326 // Recognition tasks are kept short so that latency and memory stay bounded over a long lecture.327 if Date().timeIntervalSince(startedAt) > 50 { restartLocked() }328 request?.append(buffer)329 }330}331