SPB Git

Repository Public

VoiceTranscription.swift 13.9 KB 294 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : SP Notes9// File             : Features/Voice/VoiceTranscription.swift10// Description      : Transcription of a voice note. On the device: the Speech11//                    framework on the recorded file, on-device recognition only12//                    (nothing leaves the device), word timings grouped into13//                    phrases with their start time. On the server: the paid job14//                    of `AudioRepository`, behind a protocol so the flow is15//                    tested against the fake backend.16// Module           : Features17// Repository       : https://git.spboucher.ai/sp-notes-ios18// Created          : 2026-11-1919// Last Modified    : 2026-11-1920//21// =============================================================================22// AUTHOR & PROFESSIONAL AFFILIATION23// =============================================================================24//25// Author           : Simon-Pierre Boucher26// Position         : Professor of Finance27// Institution      : Université du Québec en Outaouais (UQO)28// Department       : Département des sciences administratives29// Organization     : SP Boucher30//31// Website          : https://spboucher.ai32// Git              : https://git.spboucher.ai33//34// =============================================================================35// OFFICIAL CONTACT DIRECTORY36// =============================================================================37//38// General          : info@spboucher.ai39// Professional     : contact@spboucher.ai40// Direct           : simon@spboucher.ai41// Research         : research@spboucher.ai42// Development      : dev@spboucher.ai43// Support          : support@spboucher.ai44// Security/Admin   : admin@spboucher.ai45// Notifications    : notifications@spboucher.ai46//47// Preferred Contact: support@spboucher.ai48//49// =============================================================================50// COPYRIGHT & INTELLECTUAL PROPERTY51// =============================================================================52//53// Copyright (c) 202654// Simon-Pierre Boucher55//56// All rights reserved where applicable.57//58// This file is governed by the applicable project license,59// copyright notices, institutional agreements and relevant law.60//61// Unless expressly authorized by the applicable license or62// rights holder, copying, redistribution, sublicensing,63// publication and commercial exploitation are not permitted.64//65// Existing third-party rights and notices remain unaffected.66//67// =============================================================================68// CONFIDENTIALITY & SECURITY69// =============================================================================70//71// Classification   : Public72// Distribution     : Public repository; use is governed by the project license73// Security Contact : admin@spboucher.ai74//75// Access to this source does not independently grant permission76// to redistribute or disclose confidential materials.77//78// Do not embed credentials, private keys, access tokens,79// certificates or confidential personal information.80//81// =============================================================================82// LICENSE & MAINTENANCE83// =============================================================================84//85// License          : None specified (no license file in this repository)86// Maintainer       : Simon-Pierre Boucher87// Technical Contact: dev@spboucher.ai88//89// =============================================================================90// END OF SP BOUCHER SOURCE HEADER91// =============================================================================9293import Foundation94import SPNotesCore95import Speech9697// MARK: - Segmentation (pure)9899/// One recognised word (or short run of words) with its timing, as the Speech framework reports it.100struct SpeechPiece: Equatable, Sendable {101    var text: String102    /// Seconds from the start of the audio.103    var start: Double104    var duration: Double = 0105}106107/// Groups recognised words into the phrases of a transcript: a phrase ends at a sentence mark, at a silence, or when it gets long.108enum TranscriptSegmenter {109    static let pause = 0.9110    static let maxWords = 18111    static let maxSeconds = 12.0112113    static func segments(from pieces: [SpeechPiece]) -> [TranscriptSegment] {114        var result: [TranscriptSegment] = []115        var words: [String] = []116        var start = 0.0, lastEnd = 0.0117        func flush() {118            let text = words.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines)119            if !text.isEmpty { result.append(TranscriptSegment(t: (start * 100).rounded() / 100, text: text)) }120            words = []121        }122        for piece in pieces.sorted(by: { $0.start < $1.start }) {123            let text = piece.text.trimmingCharacters(in: .whitespacesAndNewlines)124            guard !text.isEmpty else { continue }125            if !words.isEmpty, piece.start - lastEnd >= pause || words.count >= maxWords || piece.start - start >= maxSeconds { flush() }126            if words.isEmpty { start = max(0, piece.start) }127            words.append(text)128            lastEnd = piece.start + max(0, piece.duration)129            if let last = text.last, ".!?…".contains(last) { flush() }130        }131        flush()132        return result133    }134}135136// MARK: - On device137138enum OnDeviceTranscriptionError: LocalizedError, Equatable {139    /// The user refused speech recognition (`restricted`: Settings cannot change it).140    case denied(restricted: Bool)141    /// No on-device model for this language on this device.142    case unavailable(language: String)143    case noSpeech144    case failed(String)145146    var errorDescription: String? {147        switch self {148        case .denied(let restricted):149            return restricted ? String(localized: "La reconnaissance vocale est restreinte sur cet appareil.")150                : String(localized: "SP Notes n’a pas accès à la reconnaissance vocale. Tu peux l’autoriser dans Réglages.")151        case .unavailable(let language): return String(localized: "La transcription sur l’appareil n’est pas disponible pour cette langue sur cet appareil (\(language)).")152        case .noSpeech: return String(localized: "Aucune parole n’a été reconnue dans cet enregistrement.")153        case .failed(let reason): return String(localized: "La transcription a échoué (\(reason)).")154        }155    }156}157158/// The truthful name of the engine written in `transcriptEngine` for a transcript made here.159enum TranscriptEngines {160    static let onDevice = "apple-speech-on-device"161    static func label(_ engine: String?) -> String {162        guard let engine, !engine.isEmpty else { return String(localized: "Transcription") }163        if engine == onDevice { return String(localized: "Transcription sur l’appareil") }164        return String(localized: "Transcription du serveur · \(engine)")165    }166}167168protocol SpeechTranscribing: Sendable {169    /// The locale that would be used and whether an on-device model exists for it.170    func availability() async -> (language: String, available: Bool)171    func transcribe(fileURL: URL) async throws -> [SpeechPiece]172}173174/// `SFSpeechRecognizer` on a file, on-device recognition required: when the device has no model for the language the request is175/// refused (`unavailable`) rather than sent to Apple's servers.176struct OnDeviceTranscriber: SpeechTranscribing {177    /// The device language first, then French (Canada, France) and English.178    static func preferredLocales(current: Locale = .current) -> [Locale] {179        var identifiers = [current.identifier(.bcp47)]180        for fallback in ["fr-CA", "fr-FR", "en-CA", "en-US"] where !identifiers.contains(fallback) { identifiers.append(fallback) }181        return identifiers.map(Locale.init(identifier:))182    }183184    private static func recognizer() -> SFSpeechRecognizer? {185        for locale in preferredLocales() {186            if let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition, recognizer.isAvailable { return recognizer }187        }188        return nil189    }190191    func availability() async -> (language: String, available: Bool) {192        if let recognizer = Self.recognizer() { return (recognizer.locale.identifier(.bcp47), true) }193        return (Locale.current.identifier(.bcp47), false)194    }195196    func transcribe(fileURL: URL) async throws -> [SpeechPiece] {197        let status = await withCheckedContinuation { (continuation: CheckedContinuation<SFSpeechRecognizerAuthorizationStatus, Never>) in198            SFSpeechRecognizer.requestAuthorization { continuation.resume(returning: $0) }199        }200        switch status {201        case .authorized: break202        case .restricted: throw OnDeviceTranscriptionError.denied(restricted: true)203        default: throw OnDeviceTranscriptionError.denied(restricted: false)204        }205        guard let recognizer = Self.recognizer() else { throw OnDeviceTranscriptionError.unavailable(language: Locale.current.identifier(.bcp47)) }206        let request = SFSpeechURLRecognitionRequest(url: fileURL)207        request.requiresOnDeviceRecognition = true208        request.shouldReportPartialResults = false209        request.addsPunctuation = true210        return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<[SpeechPiece], Error>) in211            let once = Once()212            recognizer.recognitionTask(with: request) { result, error in213                if let result, result.isFinal {214                    let pieces = result.bestTranscription.segments.map { SpeechPiece(text: $0.substring, start: $0.timestamp, duration: $0.duration) }215                    if once.claim() { pieces.isEmpty ? continuation.resume(throwing: OnDeviceTranscriptionError.noSpeech) : continuation.resume(returning: pieces) }216                } else if let error {217                    let ns = error as NSError218                    // 1110 / 203: nothing recognisable in the audio219                    let noSpeech = (ns.domain == "kAFAssistantErrorDomain" && (ns.code == 1110 || ns.code == 203))220                    if once.claim() { continuation.resume(throwing: noSpeech ? OnDeviceTranscriptionError.noSpeech : OnDeviceTranscriptionError.failed(error.localizedDescription)) }221                }222            }223        }224    }225}226227/// A continuation guard: the recognition callback may fire again after the final result.228final class Once: @unchecked Sendable {229    private let lock = NSLock()230    private var done = false231    func claim() -> Bool { lock.withLock { if done { return false }; done = true; return true } }232}233234// MARK: - Server235236/// The paid transcription of the server, as the voice-note screen needs it.237protocol ServerTranscribing: Sendable {238    func transcribe(attachmentId: String, summarize: Bool, force: Bool) async throws -> AudioTranscription239    func wait(attachmentId: String) async throws -> AudioTranscription240}241242struct RepositoryTranscriber: ServerTranscribing {243    let audio: AudioRepository244    var interval: Duration = .seconds(2.5)245    var timeout: Duration = .seconds(600)246247    func transcribe(attachmentId: String, summarize: Bool, force: Bool) async throws -> AudioTranscription {248        try await audio.transcribe(attachmentId: attachmentId, summarize: summarize, force: force)249    }250251    func wait(attachmentId: String) async throws -> AudioTranscription {252        try await audio.waitForTranscription(attachmentId: attachmentId, interval: interval, timeout: timeout)253    }254}255256extension AudioBlock {257    /// Writes what the server produced into the node (API-NEXT-AI §6.2: the client writes the node, through a normal save).258    /// Unknown attributes of the node are untouched.259    mutating func apply(_ transcription: AudioTranscription, using audio: AudioRepository? = nil) {260        let segments = (transcription.transcript ?? []).map { TranscriptSegment(t: $0.t, text: $0.text) }261        transcript = segments262        transcriptStatus = TranscriptStatus(rawValue: transcription.status)263        transcriptEngine = transcription.engine.map { $0 + (transcription.model.map { ":" + $0 } ?? "") }264        if let duration = transcription.duration, duration > 0, self.duration <= 0 { self.duration = duration }265        if let summary = transcription.summary, let text = summary.text, !text.isEmpty {266            var object: [String: JSONValue] = ["text": .string(text),267                                               "decisions": .array((summary.decisions ?? []).map(JSONValue.string)),268                                               "tasks": .array((summary.tasks ?? []).map(JSONValue.string))]269            object["model"] = summary.model.map(JSONValue.string) ?? .null270            object["at"] = summary.at.map { .string(VoiceNoteIO.iso($0)) } ?? .null271            self[attr: "summary"] = .object(object)272        }273    }274275    /// Writes a transcript made on the device.276    mutating func applyOnDevice(_ segments: [TranscriptSegment]) {277        transcript = segments278        transcriptStatus = .done279        transcriptEngine = TranscriptEngines.onDevice280    }281282    /// The Markdown inserted below the block when the user asks for the summary: the text, then the decisions.283    static func summaryMarkdown(_ summary: AudioSummary) -> String {284        var parts = [summary.text.trimmingCharacters(in: .whitespacesAndNewlines)]285        if !summary.decisions.isEmpty { parts.append(summary.decisions.map { "- " + $0 }.joined(separator: "\n")) }286        return parts.filter { !$0.isEmpty }.joined(separator: "\n\n")287    }288289    /// The Markdown inserted when the user asks for the tasks: a task list.290    static func tasksMarkdown(_ summary: AudioSummary) -> String {291        summary.tasks.map { "- [ ] " + $0.replacingOccurrences(of: "\n", with: " ") }.joined(separator: "\n")292    }293}294
Files main
Menu

Theme