SPB Git

Repository Public

EditorModel.swift 20.4 KB 411 lines swift

1// SP Lecture Studio2// File: Apps/LectureStudio/Shared/Editor/EditorModel.swift3// Description: Editing studio state: nondestructive edit history, suggestions, preview rendering and audio playback through the export pipeline.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Observation9import AVFoundation10import SwiftUI11import LectureCore12import LectureStorage13import LecturePDF14import LectureRender15import LectureTranscription16import LectureSync1718/// Plays the edited audio by pulling buffers from the same `EditedAudioRenderer` the exporter uses.19final class PreviewAudioPlayer {20    private let engine = AVAudioEngine()21    private let node = AVAudioPlayerNode()22    private let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 1, interleaved: false)!23    private let queue = DispatchQueue(label: "ai.spboucher.lecture.preview.audio")24    private var renderer: EditedAudioRenderer?25    private var startFrame: Int64 = 026    private var nextFrame: Int64 = 027    private var endFrame: Int64 = 028    private var generation = 029    private(set) var isPlaying = false30    var onFinished: (() -> Void)?3132    init() {33        engine.attach(node)34        engine.connect(node, to: engine.mainMixerNode, format: format)35    }3637    func play(renderer: EditedAudioRenderer, from start: MediaTime, until end: MediaTime? = nil) {38        stop()39        #if os(iOS)40        try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .default)41        try? AVAudioSession.sharedInstance().setActive(true)42        #endif43        self.renderer = renderer44        renderer.reset()45        startFrame = min(max(0, start.frames(atSampleRate: 48_000)), renderer.totalFrames)46        endFrame = min(renderer.totalFrames, end?.frames(atSampleRate: 48_000) ?? renderer.totalFrames)47        nextFrame = startFrame48        generation += 149        guard endFrame > startFrame else { onFinished?(); return }50        do { if !engine.isRunning { try engine.start() } } catch { return }51        isPlaying = true52        let gen = generation53        queue.async { [weak self] in self?.schedule(gen); self?.schedule(gen); self?.schedule(gen) }54        node.play()55    }5657    private func schedule(_ gen: Int) {58        guard gen == generation, let renderer, nextFrame < endFrame else { return }59        let n = Int(min(Int64(12_000), endFrame - nextFrame))60        let samples = renderer.render(start: nextFrame, count: n)61        guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(n)) else { return }62        buffer.frameLength = AVAudioFrameCount(n)63        samples.withUnsafeBufferPointer { buffer.floatChannelData![0].update(from: $0.baseAddress!, count: n) }64        nextFrame += Int64(n)65        let last = nextFrame >= endFrame66        node.scheduleBuffer(buffer, completionCallbackType: .dataPlayedBack) { [weak self] _ in67            guard let self else { return }68            self.queue.async {69                guard gen == self.generation else { return }70                if last { DispatchQueue.main.async { if gen == self.generation { self.isPlaying = false; self.onFinished?() } } } else { self.schedule(gen) }71            }72        }73    }7475    /// Output-timeline position of the sound currently heard.76    var currentTime: MediaTime? {77        guard isPlaying, let render = node.lastRenderTime, let t = node.playerTime(forNodeTime: render) else { return nil }78        return MediaTime(frames: startFrame + max(0, t.sampleTime), sampleRate: 48_000)79    }8081    func stop() {82        generation += 183        if node.isPlaying { node.stop() }84        isPlaying = false85    }86}8788enum EditorPanel: String, CaseIterable, Identifiable {89    case cuts, audio, chapters, transcript, ink90    var id: String { rawValue }91    var title: LocalizedStringKey {92        switch self { case .cuts: "Coupes"; case .audio: "Audio"; case .chapters: "Chapitres"; case .transcript: "Texte"; case .ink: "Encre" }93    }94    var symbol: String {95        switch self { case .cuts: "scissors"; case .audio: "waveform"; case .chapters: "list.bullet.indent"; case .transcript: "captions.bubble"; case .ink: "scribble.variable" }96    }97}9899enum TranscriptionChoice: String, CaseIterable, Identifiable { case onDevice, remote; var id: String { rawValue } }100101@MainActor @Observable102final class EditorModel {103    let session: ProjectSession104    let settings: AppSettings105    let package: ProjectPackage106    let manifest: ProjectManifest107    let document: SlideDocument108    let events: [TimelineEvent]109    let duration: MediaTime110111    private(set) var history: EditHistory112    private(set) var index: TimelineIndex113    private(set) var map: TimeMap114    private(set) var envelope: Envelope?115    var envelopeProgress: Double?116    var playhead: MediaTime = .zero { didSet { if playhead != oldValue { requestPreview() } } }117    var selection: TimeRange?118    var pixelsPerSecond: Double = 8119    @ObservationIgnored private var didFitTimeline = false120    /// First layout: show the whole recording across the available width.121    func fitTimelineIfNeeded(width: Double) {122        guard !didFitTimeline, width > 150, duration > .zero else { return }123        didFitTimeline = true124        pixelsPerSecond = min(240, max(0.3, (width - 44) / duration.seconds))125    }126    func fitTimeline(width: Double) { didFitTimeline = false; fitTimelineIfNeeded(width: width) }127    private(set) var isPlaying = false128    private(set) var previewImage: CGImage?129    var transcript: Transcript?130    var transcribing: Double?131    var transcriptionError: String?132    var statusMessage: String?133    var selectedCut: UUID?134135    @ObservationIgnored private let player = PreviewAudioPlayer()136    @ObservationIgnored private let renderQueue = DispatchQueue(label: "ai.spboucher.lecture.preview.render", qos: .userInitiated)137    @ObservationIgnored private var compositor: FrameCompositor138    @ObservationIgnored private var renderPending = false139    @ObservationIgnored private var renderBusy = false140    @ObservationIgnored private var playTimer: Timer?141    @ObservationIgnored private var playMap: TimeMap?142    @ObservationIgnored private var transcribeTask: Task<Void, Never>?143    @ObservationIgnored private var measuredGain: (key: String, gain: Float)?144145    var edits: EditDecisionList { history.current }146    var canUndo: Bool { history.canUndo }147    var canRedo: Bool { history.canRedo }148    var hasCamera: Bool { !manifest.segments(.camera).isEmpty }149150    init(session: ProjectSession, settings: AppSettings) {151        self.session = session; self.settings = settings152        package = session.package; manifest = session.manifest; document = session.document153        events = package.loadEvents().events154        duration = manifest.duration155        let h = package.loadEdits()156        history = h157        let idx = TimelineIndex(events: events, adjustments: h.current.ink)158        index = idx159        map = h.current.timeMap(duration: manifest.duration)160        envelope = package.loadEnvelope()161        transcript = package.loadTranscript()162        var s = RenderSettings(width: 960, height: 540); s.fps = 30163        compositor = FrameCompositor(document: document, index: idx, settings: s)164        player.onFinished = { [weak self] in self?.playbackEnded() }165        requestPreview()166        if envelope == nil || abs((envelope?.duration.seconds ?? 0) - duration.seconds) > 1 { computeEnvelope() }167    }168169    func teardown() { stop(); transcribeTask?.cancel() }170171    // MARK: Edits172173    func apply(_ change: (inout EditDecisionList) -> Void) {174        let before = history.current175        history.apply(change)176        guard history.current != before else { return }177        afterHistoryChange(inkChanged: before.ink != history.current.ink)178    }179    func undo() { let before = history.current.ink; history.undo(); afterHistoryChange(inkChanged: before != history.current.ink) }180    func redo() { let before = history.current.ink; history.redo(); afterHistoryChange(inkChanged: before != history.current.ink) }181    func revertToOriginal() { let before = history.current.ink; history.revertToOriginal(); afterHistoryChange(inkChanged: before != history.current.ink) }182183    private func afterHistoryChange(inkChanged: Bool) {184        try? package.saveEdits(history)185        map = edits.timeMap(duration: duration)186        if inkChanged {187            let idx = TimelineIndex(events: events, adjustments: edits.ink)188            index = idx189            renderQueue.async { [compositor] in compositor.update(index: idx) }190        }191        if isPlaying { stop() }192        requestPreview()193    }194195    var outputDuration: MediaTime { map.outputDuration }196197    func cutSelection() {198        guard let sel = selection, !sel.isEmpty else { return }199        apply { $0.addCut(sel) }200        selection = nil; playhead = min(sel.start, duration)201    }202    func muteSelection() { guard let sel = selection, !sel.isEmpty else { return }; apply { $0.muted.append(sel) }; selection = nil }203    func splitAtPlayhead() { apply { $0.split(at: playhead) } }204    func trimStartHere() { apply { $0.trimStart(to: playhead) } }205    func trimEndHere() { apply { $0.trimEnd(from: playhead, duration: duration) } }206207    /// Ripple delete: removes the clip under the playhead; everything after moves earlier.208    func deleteClipAtPlayhead() {209        guard let clip = edits.clips(duration: duration).first(where: { $0.range.contains(playhead) && !$0.isCut }) else { return }210        apply { $0.addCut(clip.range) }211    }212213    func markIn() { selection = TimeRange(start: playhead, end: max(playhead, selection?.end ?? playhead)) }214    func markOut() { selection = TimeRange(start: min(selection?.start ?? .zero, playhead), end: playhead) }215216    func restore(_ cut: Cut) { apply { $0.removeCut(cut.id) } }217    func approve(_ cut: Cut) { apply { $0.approve(cut.id) } }218    func reject(_ cut: Cut) { apply { $0.reject(cut.id) } }219    func approveAll(minConfidence: Double) {220        apply { e in for s in e.suggestions where (s.confidence ?? 0) >= minConfidence { e.approve(s.id) } }221    }222223    // MARK: Deterministic analysis224225    private func computeEnvelope() {226        guard !manifest.segments(.audio).isEmpty else { return }227        envelopeProgress = 0228        let pkg = package, m = manifest229        Task.detached(priority: .userInitiated) { [weak self] in230            let reader = TimelineAudioReader(package: pkg, manifest: m)231            let env = reader.envelope { p in Task { @MainActor [weak self] in self?.envelopeProgress = p }; return true }232            try? pkg.saveEnvelope(env)233            await MainActor.run { [weak self] in self?.envelope = env; self?.envelopeProgress = nil }234        }235    }236237    func detectSilences() {238        guard let envelope else { statusMessage = String(localized: "La forme d'onde est encore en cours d'analyse."); return }239        let result = SilenceDetector.detect(envelope, config: settings.stored.silence)240        guard result.reliable else {241            statusMessage = String(localized: "La voix et le bruit de fond sont trop proches pour repérer les silences de façon fiable. Aucune coupe n'est proposée.")242            apply { $0.setSuggestions([], origin: .silence) }; return243        }244        let cuts = SilenceDetector.suggestions(envelope, config: settings.stored.silence)245        apply { $0.setSuggestions(cuts, origin: .silence) }246        let pending = edits.suggestions.filter { $0.origin == .silence }.count247        statusMessage = pending == 0 ? String(localized: "Aucun long silence à proposer.") : String(localized: "\(pending) long(s) silence(s) proposé(s). Écoutez puis approuvez ou rejetez chacun.")248    }249250    func suggestRetakes() {251        let cuts = RetakeSuggester.suggestions(index: index, envelope: envelope, duration: duration)252        apply { $0.setSuggestions(cuts, origin: .retake) }253        statusMessage = cuts.isEmpty ? String(localized: "Aucun marqueur de reprise dans cet enregistrement.") : String(localized: "\(cuts.count) reprise(s) proposée(s) à partir de vos marqueurs.")254    }255256    func generateChapters() {257        let doc = document258        let chapters = ChapterGenerator.fromSlides(index: index, duration: duration) { page in259            doc.titleGuess(page: page) ?? String(localized: "Page \(page + 1)")260        }261        apply { $0.chapters = chapters }262    }263    func addChapterAtPlayhead() {264        let page = index.page(at: playhead)265        apply { $0.chapters.append(Chapter(time: playhead, title: document.titleGuess(page: page) ?? String(localized: "Page \(page + 1)"))); $0.chapters.sort { $0.time < $1.time } }266    }267268    // MARK: Preview269270    private func requestPreview() {271        renderPending = true272        guard !renderBusy else { return }273        renderBusy = true274        renderPending = false275        let t = playhead, compositor = compositor276        let placement = edits.cameraPlacement277        renderQueue.async { [weak self] in278            let image = compositor.image(at: t, cameraPlacement: placement)279            DispatchQueue.main.async {280                guard let self else { return }281                self.previewImage = image282                self.renderBusy = false283                if self.renderPending { self.requestPreview() }284            }285        }286    }287288    private func makeRenderer(edits e: EditDecisionList) -> EditedAudioRenderer {289        let reader = TimelineAudioReader(package: package, manifest: manifest)290        var processing = AudioProcessing()291        processing.noiseReduction = e.noiseReduction292        if let env = envelope { processing.expanderThresholdDB = min(-35, env.percentile(0.03) + 8) }293        let m = e.timeMap(duration: duration)294        let r = EditedAudioRenderer(reader: reader, map: m, edits: e, processing: processing)295        r.processing.gain = pow(10, Float(e.gainDB) / 20)296        return r297    }298299    /// Plays the edited programme from the playhead (or `from`) — or a variant of the edits to audition a suggestion.300    func play(from source: MediaTime? = nil, edits variant: EditDecisionList? = nil, seconds: Double? = nil) {301        stop()302        let e = variant ?? edits303        let m = e.timeMap(duration: duration)304        guard m.outputDuration > .zero else { return }305        var start = source ?? playhead306        if m.toOutput(start) >= m.outputDuration - MediaTime(milliseconds: 50) { start = .zero }307        let renderer = makeRenderer(edits: e)308        // Loudness normalization needs a measurement pass; reuse it while the edits do not change.309        let key = String(describing: e.cuts.map(\.range)) + "\(e.normalizeLoudness)\(e.noiseReduction)\(e.gainDB)\(e.muted)"310        if e.normalizeLoudness || e.gainDB > 0 {311            if let cached = measuredGain, cached.key == key { renderer.processing.gain = cached.gain }312            else { let g = EditedAudioRenderer.resolveGain(edits: e, measured: renderer.measure()); measuredGain = (key, g); renderer.processing.gain = g }313        }314        let outStart = m.toOutput(start)315        playMap = m316        player.play(renderer: renderer, from: outStart, until: seconds.map { outStart + MediaTime(seconds: $0) })317        isPlaying = player.isPlaying318        playTimer?.invalidate()319        playTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 30, repeats: true) { [weak self] _ in320            Task { @MainActor [weak self] in321                guard let self, self.isPlaying, let t = self.player.currentTime, let m = self.playMap else { return }322                self.playhead = m.toSource(min(t, m.outputDuration))323            }324        }325    }326327    func stop() { player.stop(); playTimer?.invalidate(); playTimer = nil; isPlaying = false }328    private func playbackEnded() { playTimer?.invalidate(); playTimer = nil; isPlaying = false }329    func togglePlay() { if isPlaying { stop() } else { play() } }330331    /// Before / after audition of a suggested or applied cut: a few seconds around the join, with or without the cut.332    func audition(_ cut: Cut, applied: Bool) {333        var e = edits334        e.cuts.removeAll { $0.id == cut.id }; e.suggestions.removeAll { $0.id == cut.id }335        if applied { e.cuts.append(cut) }336        let lead = MediaTime(seconds: 2.5)337        let start = max(.zero, cut.range.start - lead)338        play(from: start, edits: e, seconds: applied ? 5.5 : min(60, cut.range.duration.seconds + 5))339    }340341    func seek(to t: MediaTime) { let wasPlaying = isPlaying; stop(); playhead = min(max(.zero, t), duration); if wasPlaying { play() } }342343    /// Jumps to the next / previous boundary (cut edge, marker, slide change, chapter).344    func jump(forward: Bool) {345        var points = Set<MediaTime>([.zero, duration])346        for c in edits.cuts { points.insert(c.range.start); points.insert(c.range.end) }347        for s in edits.suggestions { points.insert(s.range.start) }348        for m in index.markers { points.insert(m.t) }349        for p in index.pageChanges { points.insert(p.t) }350        for c in edits.chapters { points.insert(c.time) }351        let sorted = points.sorted()352        let eps = MediaTime(milliseconds: 20)353        if forward { if let n = sorted.first(where: { $0 > playhead + eps }) { seek(to: n) } }354        else if let p = sorted.last(where: { $0 < playhead - eps }) { seek(to: p) }355    }356357    // MARK: Transcript358359    func transcribe(using choice: TranscriptionChoice) {360        guard transcribing == nil else { return }361        transcriptionError = nil362        let locale = settings.locale363        let provider: any TranscriptionProvider364        switch choice {365        case .onDevice: provider = AppleOnDeviceSpeechProvider()366        case .remote:367            guard settings.stored.remoteTranscriptionConsent else { transcriptionError = String(localized: "Activez d'abord le consentement à l'envoi de l'audio dans les réglages."); return }368            provider = RemoteTranscriptionProvider(configuration: settings.stored.remoteTranscription, token: KeychainStore.get(account: AppSettings.remoteTokenAccount))369        }370        transcribing = 0371        let pkg = package, m = manifest372        transcribeTask = Task { [weak self] in373            do {374                if choice == .onDevice, AppleOnDeviceSpeechProvider.authorization() == .notDetermined { _ = await AppleOnDeviceSpeechProvider.requestAuthorization() }375                let t = try await ProjectTranscriber.transcribe(package: pkg, manifest: m, provider: provider, locale: locale) { p in376                    Task { @MainActor [weak self] in self?.transcribing = p }377                }378                try? pkg.saveTranscript(t)379                self?.transcript = t; self?.transcribing = nil380                if t.segments.isEmpty { self?.transcriptionError = String(localized: "Aucune parole n'a été reconnue.") }381            } catch let e as TranscriptionError {382                self?.transcribing = nil383                self?.transcriptionError = Self.explain(e)384            } catch { self?.transcribing = nil; self?.transcriptionError = error.localizedDescription }385        }386    }387    func cancelTranscription() { transcribeTask?.cancel(); transcribing = nil }388389    static func explain(_ e: TranscriptionError) -> String {390        switch e {391        case .unavailable("permission"): String(localized: "Autorisez la reconnaissance vocale dans les réglages du système.")392        case .unavailable("modelNotInstalled"): String(localized: "Le modèle de reconnaissance sur l'appareil n'est pas installé pour cette langue. Installez-le dans les réglages du système (Clavier › Dictée), ou choisissez une autre langue. Rien n'a été envoyé.")393        case .unavailable("unsupportedLocale"): String(localized: "Cette langue n'est pas prise en charge par la reconnaissance vocale du système.")394        case .unavailable("notConfigured"): String(localized: "Aucun serveur de transcription n'est configuré dans les réglages.")395        case .unavailable("insecureEndpoint"): String(localized: "L'adresse du serveur doit commencer par https://.")396        case .unavailable("noAudio"): String(localized: "Ce projet ne contient pas d'audio à transcrire.")397        case .unavailable: String(localized: "La transcription n'est pas disponible pour le moment.")398        case .cancelled: String(localized: "Transcription annulée.")399        case .server(let code, let m): String(localized: "Le serveur a répondu \(code). \(m)")400        case .failed(let m): m401        }402    }403404    func updateSegment(_ id: UUID, text: String) {405        guard var t = transcript, let i = t.segments.firstIndex(where: { $0.id == id }), t.segments[i].text != text else { return }406        t.segments[i].text = text; t.segments[i].edited = true407        transcript = t; try? package.saveTranscript(t)408    }409    func deleteTranscript() { transcript = nil; try? package.saveTranscript(nil) }410}411
Files main
Menu

Theme