Repository Public
1// SP Lecture Studio2// File: Apps/LectureStudio/Shared/Editor/EditorModel.swift3// Description: Editing studio state: nondestructive edit history, suggestions, preview rendering and audio playback through the export pipeline.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Observation9import AVFoundation10import SwiftUI11import LectureCore12import LectureStorage13import LecturePDF14import LectureRender15import LectureTranscription16import LectureSync1718/// Plays the edited audio by pulling buffers from the same `EditedAudioRenderer` the exporter uses.19final class PreviewAudioPlayer {20 private let engine = AVAudioEngine()21 private let node = AVAudioPlayerNode()22 private let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 1, interleaved: false)!23 private let queue = DispatchQueue(label: "ai.spboucher.lecture.preview.audio")24 private var renderer: EditedAudioRenderer?25 private var startFrame: Int64 = 026 private var nextFrame: Int64 = 027 private var endFrame: Int64 = 028 private var generation = 029 private(set) var isPlaying = false30 var onFinished: (() -> Void)?3132 init() {33 engine.attach(node)34 engine.connect(node, to: engine.mainMixerNode, format: format)35 }3637 func play(renderer: EditedAudioRenderer, from start: MediaTime, until end: MediaTime? = nil) {38 stop()39 #if os(iOS)40 try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .default)41 try? AVAudioSession.sharedInstance().setActive(true)42 #endif43 self.renderer = renderer44 renderer.reset()45 startFrame = min(max(0, start.frames(atSampleRate: 48_000)), renderer.totalFrames)46 endFrame = min(renderer.totalFrames, end?.frames(atSampleRate: 48_000) ?? renderer.totalFrames)47 nextFrame = startFrame48 generation += 149 guard endFrame > startFrame else { onFinished?(); return }50 do { if !engine.isRunning { try engine.start() } } catch { return }51 isPlaying = true52 let gen = generation53 queue.async { [weak self] in self?.schedule(gen); self?.schedule(gen); self?.schedule(gen) }54 node.play()55 }5657 private func schedule(_ gen: Int) {58 guard gen == generation, let renderer, nextFrame < endFrame else { return }59 let n = Int(min(Int64(12_000), endFrame - nextFrame))60 let samples = renderer.render(start: nextFrame, count: n)61 guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(n)) else { return }62 buffer.frameLength = AVAudioFrameCount(n)63 samples.withUnsafeBufferPointer { buffer.floatChannelData![0].update(from: $0.baseAddress!, count: n) }64 nextFrame += Int64(n)65 let last = nextFrame >= endFrame66 node.scheduleBuffer(buffer, completionCallbackType: .dataPlayedBack) { [weak self] _ in67 guard let self else { return }68 self.queue.async {69 guard gen == self.generation else { return }70 if last { DispatchQueue.main.async { if gen == self.generation { self.isPlaying = false; self.onFinished?() } } } else { self.schedule(gen) }71 }72 }73 }7475 /// Output-timeline position of the sound currently heard.76 var currentTime: MediaTime? {77 guard isPlaying, let render = node.lastRenderTime, let t = node.playerTime(forNodeTime: render) else { return nil }78 return MediaTime(frames: startFrame + max(0, t.sampleTime), sampleRate: 48_000)79 }8081 func stop() {82 generation += 183 if node.isPlaying { node.stop() }84 isPlaying = false85 }86}8788enum EditorPanel: String, CaseIterable, Identifiable {89 case cuts, audio, chapters, transcript, ink90 var id: String { rawValue }91 var title: LocalizedStringKey {92 switch self { case .cuts: "Coupes"; case .audio: "Audio"; case .chapters: "Chapitres"; case .transcript: "Texte"; case .ink: "Encre" }93 }94 var symbol: String {95 switch self { case .cuts: "scissors"; case .audio: "waveform"; case .chapters: "list.bullet.indent"; case .transcript: "captions.bubble"; case .ink: "scribble.variable" }96 }97}9899enum TranscriptionChoice: String, CaseIterable, Identifiable { case onDevice, remote; var id: String { rawValue } }100101@MainActor @Observable102final class EditorModel {103 let session: ProjectSession104 let settings: AppSettings105 let package: ProjectPackage106 let manifest: ProjectManifest107 let document: SlideDocument108 let events: [TimelineEvent]109 let duration: MediaTime110111 private(set) var history: EditHistory112 private(set) var index: TimelineIndex113 private(set) var map: TimeMap114 private(set) var envelope: Envelope?115 var envelopeProgress: Double?116 var playhead: MediaTime = .zero { didSet { if playhead != oldValue { requestPreview() } } }117 var selection: TimeRange?118 var pixelsPerSecond: Double = 8119 @ObservationIgnored private var didFitTimeline = false120 /// First layout: show the whole recording across the available width.121 func fitTimelineIfNeeded(width: Double) {122 guard !didFitTimeline, width > 150, duration > .zero else { return }123 didFitTimeline = true124 pixelsPerSecond = min(240, max(0.3, (width - 44) / duration.seconds))125 }126 func fitTimeline(width: Double) { didFitTimeline = false; fitTimelineIfNeeded(width: width) }127 private(set) var isPlaying = false128 private(set) var previewImage: CGImage?129 var transcript: Transcript?130 var transcribing: Double?131 var transcriptionError: String?132 var statusMessage: String?133 var selectedCut: UUID?134135 @ObservationIgnored private let player = PreviewAudioPlayer()136 @ObservationIgnored private let renderQueue = DispatchQueue(label: "ai.spboucher.lecture.preview.render", qos: .userInitiated)137 @ObservationIgnored private var compositor: FrameCompositor138 @ObservationIgnored private var renderPending = false139 @ObservationIgnored private var renderBusy = false140 @ObservationIgnored private var playTimer: Timer?141 @ObservationIgnored private var playMap: TimeMap?142 @ObservationIgnored private var transcribeTask: Task<Void, Never>?143 @ObservationIgnored private var measuredGain: (key: String, gain: Float)?144145 var edits: EditDecisionList { history.current }146 var canUndo: Bool { history.canUndo }147 var canRedo: Bool { history.canRedo }148 var hasCamera: Bool { !manifest.segments(.camera).isEmpty }149150 init(session: ProjectSession, settings: AppSettings) {151 self.session = session; self.settings = settings152 package = session.package; manifest = session.manifest; document = session.document153 events = package.loadEvents().events154 duration = manifest.duration155 let h = package.loadEdits()156 history = h157 let idx = TimelineIndex(events: events, adjustments: h.current.ink)158 index = idx159 map = h.current.timeMap(duration: manifest.duration)160 envelope = package.loadEnvelope()161 transcript = package.loadTranscript()162 var s = RenderSettings(width: 960, height: 540); s.fps = 30163 compositor = FrameCompositor(document: document, index: idx, settings: s)164 player.onFinished = { [weak self] in self?.playbackEnded() }165 requestPreview()166 if envelope == nil || abs((envelope?.duration.seconds ?? 0) - duration.seconds) > 1 { computeEnvelope() }167 }168169 func teardown() { stop(); transcribeTask?.cancel() }170171 // MARK: Edits172173 func apply(_ change: (inout EditDecisionList) -> Void) {174 let before = history.current175 history.apply(change)176 guard history.current != before else { return }177 afterHistoryChange(inkChanged: before.ink != history.current.ink)178 }179 func undo() { let before = history.current.ink; history.undo(); afterHistoryChange(inkChanged: before != history.current.ink) }180 func redo() { let before = history.current.ink; history.redo(); afterHistoryChange(inkChanged: before != history.current.ink) }181 func revertToOriginal() { let before = history.current.ink; history.revertToOriginal(); afterHistoryChange(inkChanged: before != history.current.ink) }182183 private func afterHistoryChange(inkChanged: Bool) {184 try? package.saveEdits(history)185 map = edits.timeMap(duration: duration)186 if inkChanged {187 let idx = TimelineIndex(events: events, adjustments: edits.ink)188 index = idx189 renderQueue.async { [compositor] in compositor.update(index: idx) }190 }191 if isPlaying { stop() }192 requestPreview()193 }194195 var outputDuration: MediaTime { map.outputDuration }196197 func cutSelection() {198 guard let sel = selection, !sel.isEmpty else { return }199 apply { $0.addCut(sel) }200 selection = nil; playhead = min(sel.start, duration)201 }202 func muteSelection() { guard let sel = selection, !sel.isEmpty else { return }; apply { $0.muted.append(sel) }; selection = nil }203 func splitAtPlayhead() { apply { $0.split(at: playhead) } }204 func trimStartHere() { apply { $0.trimStart(to: playhead) } }205 func trimEndHere() { apply { $0.trimEnd(from: playhead, duration: duration) } }206207 /// Ripple delete: removes the clip under the playhead; everything after moves earlier.208 func deleteClipAtPlayhead() {209 guard let clip = edits.clips(duration: duration).first(where: { $0.range.contains(playhead) && !$0.isCut }) else { return }210 apply { $0.addCut(clip.range) }211 }212213 func markIn() { selection = TimeRange(start: playhead, end: max(playhead, selection?.end ?? playhead)) }214 func markOut() { selection = TimeRange(start: min(selection?.start ?? .zero, playhead), end: playhead) }215216 func restore(_ cut: Cut) { apply { $0.removeCut(cut.id) } }217 func approve(_ cut: Cut) { apply { $0.approve(cut.id) } }218 func reject(_ cut: Cut) { apply { $0.reject(cut.id) } }219 func approveAll(minConfidence: Double) {220 apply { e in for s in e.suggestions where (s.confidence ?? 0) >= minConfidence { e.approve(s.id) } }221 }222223 // MARK: Deterministic analysis224225 private func computeEnvelope() {226 guard !manifest.segments(.audio).isEmpty else { return }227 envelopeProgress = 0228 let pkg = package, m = manifest229 Task.detached(priority: .userInitiated) { [weak self] in230 let reader = TimelineAudioReader(package: pkg, manifest: m)231 let env = reader.envelope { p in Task { @MainActor [weak self] in self?.envelopeProgress = p }; return true }232 try? pkg.saveEnvelope(env)233 await MainActor.run { [weak self] in self?.envelope = env; self?.envelopeProgress = nil }234 }235 }236237 func detectSilences() {238 guard let envelope else { statusMessage = String(localized: "La forme d'onde est encore en cours d'analyse."); return }239 let result = SilenceDetector.detect(envelope, config: settings.stored.silence)240 guard result.reliable else {241 statusMessage = String(localized: "La voix et le bruit de fond sont trop proches pour repérer les silences de façon fiable. Aucune coupe n'est proposée.")242 apply { $0.setSuggestions([], origin: .silence) }; return243 }244 let cuts = SilenceDetector.suggestions(envelope, config: settings.stored.silence)245 apply { $0.setSuggestions(cuts, origin: .silence) }246 let pending = edits.suggestions.filter { $0.origin == .silence }.count247 statusMessage = pending == 0 ? String(localized: "Aucun long silence à proposer.") : String(localized: "\(pending) long(s) silence(s) proposé(s). Écoutez puis approuvez ou rejetez chacun.")248 }249250 func suggestRetakes() {251 let cuts = RetakeSuggester.suggestions(index: index, envelope: envelope, duration: duration)252 apply { $0.setSuggestions(cuts, origin: .retake) }253 statusMessage = cuts.isEmpty ? String(localized: "Aucun marqueur de reprise dans cet enregistrement.") : String(localized: "\(cuts.count) reprise(s) proposée(s) à partir de vos marqueurs.")254 }255256 func generateChapters() {257 let doc = document258 let chapters = ChapterGenerator.fromSlides(index: index, duration: duration) { page in259 doc.titleGuess(page: page) ?? String(localized: "Page \(page + 1)")260 }261 apply { $0.chapters = chapters }262 }263 func addChapterAtPlayhead() {264 let page = index.page(at: playhead)265 apply { $0.chapters.append(Chapter(time: playhead, title: document.titleGuess(page: page) ?? String(localized: "Page \(page + 1)"))); $0.chapters.sort { $0.time < $1.time } }266 }267268 // MARK: Preview269270 private func requestPreview() {271 renderPending = true272 guard !renderBusy else { return }273 renderBusy = true274 renderPending = false275 let t = playhead, compositor = compositor276 let placement = edits.cameraPlacement277 renderQueue.async { [weak self] in278 let image = compositor.image(at: t, cameraPlacement: placement)279 DispatchQueue.main.async {280 guard let self else { return }281 self.previewImage = image282 self.renderBusy = false283 if self.renderPending { self.requestPreview() }284 }285 }286 }287288 private func makeRenderer(edits e: EditDecisionList) -> EditedAudioRenderer {289 let reader = TimelineAudioReader(package: package, manifest: manifest)290 var processing = AudioProcessing()291 processing.noiseReduction = e.noiseReduction292 if let env = envelope { processing.expanderThresholdDB = min(-35, env.percentile(0.03) + 8) }293 let m = e.timeMap(duration: duration)294 let r = EditedAudioRenderer(reader: reader, map: m, edits: e, processing: processing)295 r.processing.gain = pow(10, Float(e.gainDB) / 20)296 return r297 }298299 /// Plays the edited programme from the playhead (or `from`) — or a variant of the edits to audition a suggestion.300 func play(from source: MediaTime? = nil, edits variant: EditDecisionList? = nil, seconds: Double? = nil) {301 stop()302 let e = variant ?? edits303 let m = e.timeMap(duration: duration)304 guard m.outputDuration > .zero else { return }305 var start = source ?? playhead306 if m.toOutput(start) >= m.outputDuration - MediaTime(milliseconds: 50) { start = .zero }307 let renderer = makeRenderer(edits: e)308 // Loudness normalization needs a measurement pass; reuse it while the edits do not change.309 let key = String(describing: e.cuts.map(\.range)) + "\(e.normalizeLoudness)\(e.noiseReduction)\(e.gainDB)\(e.muted)"310 if e.normalizeLoudness || e.gainDB > 0 {311 if let cached = measuredGain, cached.key == key { renderer.processing.gain = cached.gain }312 else { let g = EditedAudioRenderer.resolveGain(edits: e, measured: renderer.measure()); measuredGain = (key, g); renderer.processing.gain = g }313 }314 let outStart = m.toOutput(start)315 playMap = m316 player.play(renderer: renderer, from: outStart, until: seconds.map { outStart + MediaTime(seconds: $0) })317 isPlaying = player.isPlaying318 playTimer?.invalidate()319 playTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 30, repeats: true) { [weak self] _ in320 Task { @MainActor [weak self] in321 guard let self, self.isPlaying, let t = self.player.currentTime, let m = self.playMap else { return }322 self.playhead = m.toSource(min(t, m.outputDuration))323 }324 }325 }326327 func stop() { player.stop(); playTimer?.invalidate(); playTimer = nil; isPlaying = false }328 private func playbackEnded() { playTimer?.invalidate(); playTimer = nil; isPlaying = false }329 func togglePlay() { if isPlaying { stop() } else { play() } }330331 /// Before / after audition of a suggested or applied cut: a few seconds around the join, with or without the cut.332 func audition(_ cut: Cut, applied: Bool) {333 var e = edits334 e.cuts.removeAll { $0.id == cut.id }; e.suggestions.removeAll { $0.id == cut.id }335 if applied { e.cuts.append(cut) }336 let lead = MediaTime(seconds: 2.5)337 let start = max(.zero, cut.range.start - lead)338 play(from: start, edits: e, seconds: applied ? 5.5 : min(60, cut.range.duration.seconds + 5))339 }340341 func seek(to t: MediaTime) { let wasPlaying = isPlaying; stop(); playhead = min(max(.zero, t), duration); if wasPlaying { play() } }342343 /// Jumps to the next / previous boundary (cut edge, marker, slide change, chapter).344 func jump(forward: Bool) {345 var points = Set<MediaTime>([.zero, duration])346 for c in edits.cuts { points.insert(c.range.start); points.insert(c.range.end) }347 for s in edits.suggestions { points.insert(s.range.start) }348 for m in index.markers { points.insert(m.t) }349 for p in index.pageChanges { points.insert(p.t) }350 for c in edits.chapters { points.insert(c.time) }351 let sorted = points.sorted()352 let eps = MediaTime(milliseconds: 20)353 if forward { if let n = sorted.first(where: { $0 > playhead + eps }) { seek(to: n) } }354 else if let p = sorted.last(where: { $0 < playhead - eps }) { seek(to: p) }355 }356357 // MARK: Transcript358359 func transcribe(using choice: TranscriptionChoice) {360 guard transcribing == nil else { return }361 transcriptionError = nil362 let locale = settings.locale363 let provider: any TranscriptionProvider364 switch choice {365 case .onDevice: provider = AppleOnDeviceSpeechProvider()366 case .remote:367 guard settings.stored.remoteTranscriptionConsent else { transcriptionError = String(localized: "Activez d'abord le consentement à l'envoi de l'audio dans les réglages."); return }368 provider = RemoteTranscriptionProvider(configuration: settings.stored.remoteTranscription, token: KeychainStore.get(account: AppSettings.remoteTokenAccount))369 }370 transcribing = 0371 let pkg = package, m = manifest372 transcribeTask = Task { [weak self] in373 do {374 if choice == .onDevice, AppleOnDeviceSpeechProvider.authorization() == .notDetermined { _ = await AppleOnDeviceSpeechProvider.requestAuthorization() }375 let t = try await ProjectTranscriber.transcribe(package: pkg, manifest: m, provider: provider, locale: locale) { p in376 Task { @MainActor [weak self] in self?.transcribing = p }377 }378 try? pkg.saveTranscript(t)379 self?.transcript = t; self?.transcribing = nil380 if t.segments.isEmpty { self?.transcriptionError = String(localized: "Aucune parole n'a été reconnue.") }381 } catch let e as TranscriptionError {382 self?.transcribing = nil383 self?.transcriptionError = Self.explain(e)384 } catch { self?.transcribing = nil; self?.transcriptionError = error.localizedDescription }385 }386 }387 func cancelTranscription() { transcribeTask?.cancel(); transcribing = nil }388389 static func explain(_ e: TranscriptionError) -> String {390 switch e {391 case .unavailable("permission"): String(localized: "Autorisez la reconnaissance vocale dans les réglages du système.")392 case .unavailable("modelNotInstalled"): String(localized: "Le modèle de reconnaissance sur l'appareil n'est pas installé pour cette langue. Installez-le dans les réglages du système (Clavier › Dictée), ou choisissez une autre langue. Rien n'a été envoyé.")393 case .unavailable("unsupportedLocale"): String(localized: "Cette langue n'est pas prise en charge par la reconnaissance vocale du système.")394 case .unavailable("notConfigured"): String(localized: "Aucun serveur de transcription n'est configuré dans les réglages.")395 case .unavailable("insecureEndpoint"): String(localized: "L'adresse du serveur doit commencer par https://.")396 case .unavailable("noAudio"): String(localized: "Ce projet ne contient pas d'audio à transcrire.")397 case .unavailable: String(localized: "La transcription n'est pas disponible pour le moment.")398 case .cancelled: String(localized: "Transcription annulée.")399 case .server(let code, let m): String(localized: "Le serveur a répondu \(code). \(m)")400 case .failed(let m): m401 }402 }403404 func updateSegment(_ id: UUID, text: String) {405 guard var t = transcript, let i = t.segments.firstIndex(where: { $0.id == id }), t.segments[i].text != text else { return }406 t.segments[i].text = text; t.segments[i].edited = true407 transcript = t; try? package.saveTranscript(t)408 }409 func deleteTranscript() { transcript = nil; try? package.saveTranscript(nil) }410}411