Repository Public
1// SP Lecture Studio — LectureRender2// File: Packages/LectureRender/Sources/VideoExporter.swift3// Description: Renders the edited lecture to MP4 (H.264 / HEVC + AAC) with AVAssetWriter: progress, cancellation, disk checks, atomic finish.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import AVFoundation9import CoreGraphics10import CoreVideo11import VideoToolbox12import LectureCore13import LectureStorage14import LecturePDF1516public enum ExportError: Error, Equatable, CustomStringConvertible {17 case nothingToExport18 case cancelled19 case diskFull(needed: Int64, available: Int64)20 case codecUnavailable(String)21 case writerFailed(String)2223 public var description: String {24 switch self {25 case .nothingToExport: "nothing to export"26 case .cancelled: "cancelled"27 case .diskFull(let n, let a): "not enough disk space (needs about \(n / 1_000_000) MB, \(a / 1_000_000) MB available)"28 case .codecUnavailable(let c): "\(c) encoding is not available on this device"29 case .writerFailed(let m): "the video writer failed: \(m)"30 }31 }32}3334public struct ExportResult: Sendable, Hashable {35 public var url: URL36 public var duration: MediaTime37 public var frames: Int38 /// Frames identical to the previous one (the rendered buffer was reused).39 public var reusedFrames: Int40 public var byteCount: Int6441 public var elapsed: TimeInterval42 public var measuredLUFS: Double?43 public var appliedGainDB: Double44 public var unreadableSegments: [String]45}4647/// Everything needed to render a project, loaded once. Shared by the exporter and the editor preview.48public struct RenderSource {49 public var package: ProjectPackage50 public var manifest: ProjectManifest51 public var document: SlideDocument52 public var events: [TimelineEvent]53 public var edits: EditDecisionList54 public var envelope: Envelope?5556 public init(package: ProjectPackage, manifest: ProjectManifest, document: SlideDocument, events: [TimelineEvent], edits: EditDecisionList, envelope: Envelope? = nil) {57 self.package = package; self.manifest = manifest; self.document = document; self.events = events; self.edits = edits; self.envelope = envelope58 }5960 public static func load(_ package: ProjectPackage) throws -> RenderSource {61 let manifest = try package.loadManifest()62 guard manifest.pdf != nil, FileManager.default.fileExists(atPath: package.pdfURL.path) else { throw StorageError.pdfMissing }63 return RenderSource(package: package, manifest: manifest, document: try SlideDocument(url: package.pdfURL),64 events: package.loadEvents().events, edits: package.loadEdits().current, envelope: package.loadEnvelope())65 }6667 /// Timeline length: the recorded duration, extended to the last event (PDF-only sessions have events but no audio).68 public var duration: MediaTime { manifest.duration }69 public var index: TimelineIndex { TimelineIndex(events: events, adjustments: edits.ink, initialPage: 0) }70 public var timeMap: TimeMap { edits.timeMap(duration: duration) }71}7273public final class VideoExporter {74 public let source: RenderSource75 public let settings: RenderSettings76 public static let audioRate = 48_000.07778 public init(source: RenderSource, settings: RenderSettings) { self.source = source; self.settings = settings }7980 public static func isAvailable(_ codec: VideoCodec) -> Bool {81 switch codec {82 case .h264: return true83 case .hevc: return VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC) || AVAssetExportSession.allExportPresets().contains(AVAssetExportPresetHEVCHighestQuality)84 }85 }8687 /// Exports to `destination`. The file appears there only once complete; a cancelled or failed export leaves nothing behind.88 /// - Parameters:89 /// - progress: called with 0…1 from background queues.90 /// - isCancelled: polled between frames.91 public func export(to destination: URL, progress: @escaping @Sendable (Double) -> Void = { _ in },92 isCancelled: @escaping @Sendable () -> Bool = { false }) async throws -> ExportResult {93 let started = Date()94 let map = source.timeMap95 guard map.outputDuration > .zero else { throw ExportError.nothingToExport }96 guard Self.isAvailable(settings.codec) else { throw ExportError.codecUnavailable(settings.codec.rawValue) }97 let fm = FileManager.default98 let needed = settings.estimatedBytes(duration: map.outputDuration)99 let available = ProjectPackage.freeDiskSpace(at: destination.deletingLastPathComponent())100 if available > 0, available < needed + 50_000_000 { throw ExportError.diskFull(needed: needed, available: available) }101102 // Work in a private scratch folder next to the destination (same volume, so the final move is atomic). AVAssetWriter103 // creates side files of its own there; removing the folder removes every trace of an unfinished export.104 let scratch = destination.deletingLastPathComponent().appendingPathComponent(".export.partial-\(UUID().uuidString.prefix(8))", isDirectory: true)105 try fm.createDirectory(at: scratch, withIntermediateDirectories: true)106 let partial = scratch.appendingPathComponent(destination.lastPathComponent)107 func cleanup() { try? fm.removeItem(at: scratch) }108109 // Audio: optional measurement pass, then the final gain.110 let reader = TimelineAudioReader(package: source.package, manifest: source.manifest, sampleRate: Self.audioRate)111 var processing = AudioProcessing()112 processing.noiseReduction = source.edits.noiseReduction113 if let env = source.envelope { processing.expanderThresholdDB = min(-35, env.percentile(0.03) + 8) }114 let audio = EditedAudioRenderer(reader: reader, map: map, edits: source.edits, processing: processing)115 var measured: (lufs: Double?, peak: Float) = (nil, 0)116 let needsMeasure = source.edits.normalizeLoudness || source.edits.gainDB > 0117 if needsMeasure {118 measured = audio.measure { p in progress(p * 0.08); return !isCancelled() }119 if isCancelled() { throw ExportError.cancelled }120 }121 audio.processing.gain = needsMeasure ? EditedAudioRenderer.resolveGain(edits: source.edits, measured: measured) : pow(10, Float(source.edits.gainDB) / 20)122 audio.reset()123 let base = needsMeasure ? 0.08 : 0.0124125 let writer: AVAssetWriter126 do { writer = try AVAssetWriter(outputURL: partial, fileType: .mp4) } catch { throw ExportError.writerFailed(error.localizedDescription) }127 writer.shouldOptimizeForNetworkUse = true128129 var compression: [String: Any] = [AVVideoAverageBitRateKey: settings.resolvedVideoBitrate, AVVideoExpectedSourceFrameRateKey: settings.fps,130 AVVideoMaxKeyFrameIntervalKey: settings.fps * 2]131 if settings.codec == .h264 { compression[AVVideoProfileLevelKey] = AVVideoProfileLevelH264HighAutoLevel }132 let videoSettings: [String: Any] = [133 AVVideoCodecKey: settings.codec == .hevc ? AVVideoCodecType.hevc : AVVideoCodecType.h264,134 AVVideoWidthKey: settings.width, AVVideoHeightKey: settings.height,135 AVVideoColorPropertiesKey: [AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2,136 AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2],137 AVVideoCompressionPropertiesKey: compression,138 ]139 let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)140 videoInput.expectsMediaDataInRealTime = false141 let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoInput, sourcePixelBufferAttributes: [142 kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, kCVPixelBufferWidthKey as String: settings.width,143 kCVPixelBufferHeightKey as String: settings.height, kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,144 ])145 let audioSettings: [String: Any] = [AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: Self.audioRate, AVNumberOfChannelsKey: 1,146 AVEncoderBitRateKey: min(settings.audioBitrate, 160_000)]147 let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings)148 audioInput.expectsMediaDataInRealTime = false149 guard writer.canAdd(videoInput), writer.canAdd(audioInput) else { cleanup(); throw ExportError.writerFailed("inputs rejected") }150 writer.add(videoInput); writer.add(audioInput)151 guard writer.startWriting() else { cleanup(); throw ExportError.writerFailed(writer.error?.localizedDescription ?? "cannot start") }152 writer.startSession(atSourceTime: .zero)153154 let fps = Int64(settings.fps)155 let totalFrames = Int(max(1, (Double(map.outputDuration.ns) * Double(fps) / 1e9).rounded(.up)))156 let compositor = FrameCompositor(document: source.document, index: source.index, settings: settings)157 let cameraSegments = settings.includeCamera && !source.edits.cameraHidden ? source.manifest.segments(.camera) : []158 let camera = cameraSegments.isEmpty ? nil : CameraFrameProvider(package: source.package, segments: cameraSegments)159 let placementOverride = source.edits.cameraPlacement160161 final class Shared: @unchecked Sendable { var frame = 0; var reused = 0; var audioPos: Int64 = 0; var cancelled = false; var failure: String? }162 let shared = Shared()163 let videoQueue = DispatchQueue(label: "ai.spboucher.lecture.export.video"), audioQueue = DispatchQueue(label: "ai.spboucher.lecture.export.audio")164165 async let videoDone: Void = withCheckedContinuation { (cont: CheckedContinuation<Void, Never>) in166 var lastSignature: FrameSignature?167 var lastBuffer: CVPixelBuffer?168 videoInput.requestMediaDataWhenReady(on: videoQueue) {169 while videoInput.isReadyForMoreMediaData {170 if shared.frame >= totalFrames || shared.cancelled || shared.failure != nil { videoInput.markAsFinished(); cont.resume(); return }171 if isCancelled() { shared.cancelled = true; continue }172 let i = shared.frame173 let outTime = MediaTime(ns: Int64(i) * 1_000_000_000 / fps)174 let src = map.toSource(outTime)175 let sig = compositor.signature(at: src)176 let cameraImage = camera.flatMap { sig.camera?.enabled ?? true ? $0.frame(at: src) : nil }177 var buffer: CVPixelBuffer?178 if cameraImage == nil, sig == lastSignature, let lastBuffer {179 buffer = lastBuffer; shared.reused += 1180 } else if let pool = adaptor.pixelBufferPool {181 var fresh: CVPixelBuffer?182 CVPixelBufferPoolCreatePixelBuffer(nil, pool, &fresh)183 if let fresh {184 CVPixelBufferLockBaseAddress(fresh, [])185 if let ctx = Bitmap.context(width: CVPixelBufferGetWidth(fresh), height: CVPixelBufferGetHeight(fresh),186 data: CVPixelBufferGetBaseAddress(fresh), bytesPerRow: CVPixelBufferGetBytesPerRow(fresh)) {187 compositor.render(at: src, into: ctx, camera: cameraImage, cameraPlacement: placementOverride)188 }189 CVPixelBufferUnlockBaseAddress(fresh, [])190 buffer = fresh; lastBuffer = fresh; lastSignature = sig191 }192 }193 guard let buffer, adaptor.append(buffer, withPresentationTime: CMTime(value: Int64(i), timescale: CMTimeScale(fps))) else {194 shared.failure = writer.error.map { Self.describe($0) } ?? "frame \(i) could not be appended"; continue195 }196 shared.frame += 1197 if i % 15 == 0 { progress(base + (1 - base) * Double(i) / Double(totalFrames) * 0.98) }198 }199 }200 }201202 async let audioDone: Void = withCheckedContinuation { (cont: CheckedContinuation<Void, Never>) in203 let total = audio.totalFrames204 let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.audioRate, channels: 1, interleaved: false)!205 audioInput.requestMediaDataWhenReady(on: audioQueue) {206 while audioInput.isReadyForMoreMediaData {207 if shared.audioPos >= total || shared.cancelled || shared.failure != nil { audioInput.markAsFinished(); cont.resume(); return }208 let n = Int(min(Int64(24_000), total - shared.audioPos))209 let samples = audio.render(start: shared.audioPos, count: n)210 guard let sb = Self.sampleBuffer(samples, format: format, position: shared.audioPos), audioInput.append(sb) else {211 shared.failure = writer.error.map { Self.describe($0) } ?? "audio could not be appended"; continue212 }213 shared.audioPos += Int64(n)214 }215 }216 }217 _ = await (videoDone, audioDone)218219 if shared.cancelled { writer.cancelWriting(); cleanup(); throw ExportError.cancelled }220 if let failure = shared.failure {221 let err = writer.error222 writer.cancelWriting(); cleanup()223 if let err, Self.isDiskFull(err) { throw ExportError.diskFull(needed: needed, available: ProjectPackage.freeDiskSpace(at: destination.deletingLastPathComponent())) }224 throw ExportError.writerFailed(failure)225 }226 writer.endSession(atSourceTime: CMTime(value: map.outputDuration.ns, timescale: MediaTime.timescale))227 await writer.finishWriting()228 guard writer.status == .completed else {229 let err = writer.error; cleanup()230 if let err, Self.isDiskFull(err) { throw ExportError.diskFull(needed: needed, available: 0) }231 throw ExportError.writerFailed(err.map { Self.describe($0) } ?? "unknown")232 }233 // Atomic hand-over: the destination is replaced only by a complete file.234 if fm.fileExists(atPath: destination.path) { _ = try? fm.replaceItemAt(destination, withItemAt: partial) } else { try fm.moveItem(at: partial, to: destination) }235 cleanup()236 progress(1)237 let size = (try? fm.attributesOfItem(atPath: destination.path)[.size] as? NSNumber)?.int64Value ?? 0238 return ExportResult(url: destination, duration: map.outputDuration, frames: totalFrames, reusedFrames: shared.reused, byteCount: size,239 elapsed: Date().timeIntervalSince(started), measuredLUFS: measured.lufs,240 appliedGainDB: Double(20 * log10(max(audio.processing.gain, 1e-6))), unreadableSegments: reader.unreadable)241 }242243 static func describe(_ error: Error) -> String { let e = error as NSError; return "\(e.domain) \(e.code): \(e.localizedDescription)" }244245 static func isDiskFull(_ error: Error) -> Bool {246 let e = error as NSError247 if e.domain == AVFoundationErrorDomain && e.code == AVError.diskFull.rawValue { return true }248 if e.domain == NSPOSIXErrorDomain && e.code == Int(ENOSPC) { return true }249 if e.domain == NSCocoaErrorDomain && e.code == NSFileWriteOutOfSpaceError { return true }250 if let u = e.userInfo[NSUnderlyingErrorKey] as? NSError { return isDiskFull(u) }251 return false252 }253254 static func sampleBuffer(_ samples: [Float], format: AVAudioFormat, position: Int64) -> CMSampleBuffer? {255 guard let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)) else { return nil }256 pcm.frameLength = AVAudioFrameCount(samples.count)257 samples.withUnsafeBufferPointer { pcm.floatChannelData![0].update(from: $0.baseAddress!, count: samples.count) }258 var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: CMTimeScale(format.sampleRate)),259 presentationTimeStamp: CMTime(value: position, timescale: CMTimeScale(format.sampleRate)), decodeTimeStamp: .invalid)260 var sb: CMSampleBuffer?261 guard CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, dataReady: false, makeDataReadyCallback: nil, refcon: nil,262 formatDescription: format.formatDescription, sampleCount: CMItemCount(samples.count), sampleTimingEntryCount: 1,263 sampleTimingArray: &timing, sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sb) == noErr, let sb else { return nil }264 guard CMSampleBufferSetDataBufferFromAudioBufferList(sb, blockBufferAllocator: kCFAllocatorDefault, blockBufferMemoryAllocator: kCFAllocatorDefault,265 flags: 0, bufferList: pcm.audioBufferList) == noErr else { return nil }266 return sb267 }268}269270/// Sequential reader of the camera segments: returns the camera frame shown at a timeline time.271public final class CameraFrameProvider {272 private let package: ProjectPackage273 private let segments: [MediaSegment]274 private var current: (index: Int, reader: AVAssetReader, output: AVAssetReaderTrackOutput, transform: CGAffineTransform)?275 private var held: (time: MediaTime, image: CGImage)?276 private var pending: (time: MediaTime, image: CGImage)?277278 public init(package: ProjectPackage, segments: [MediaSegment]) {279 self.package = package; self.segments = segments.sorted { $0.start < $1.start }280 }281282 public func frame(at t: MediaTime) -> CGImage? {283 guard let si = segments.firstIndex(where: { $0.range.contains(t) }) else { return nil }284 let seg = segments[si]285 let local = t - seg.start286 // Reopen when the segment changes or when time jumps backwards / far ahead (a cut).287 let needsOpen = current?.index != si || (held.map { $0.time > local } ?? false) || (held.map { (local - $0.time).seconds > 2 } ?? false && pending == nil)288 if needsOpen { open(si, from: local) }289 guard let cur = current else { return nil }290 while true {291 if let p = pending {292 if p.time <= local { held = p; pending = nil } else { break }293 }294 guard cur.reader.status == .reading, let sb = cur.output.copyNextSampleBuffer() else { break }295 guard let pb = CMSampleBufferGetImageBuffer(sb) else { continue }296 var img: CGImage?297 VTCreateCGImageFromCVPixelBuffer(pb, options: nil, imageOut: &img)298 if let img { pending = (MediaTime(seconds: CMSampleBufferGetPresentationTimeStamp(sb).seconds), img) }299 }300 return held?.image301 }302303 private func open(_ index: Int, from local: MediaTime) {304 current?.reader.cancelReading(); current = nil; held = nil; pending = nil305 let asset = AVURLAsset(url: package.url(segments[index].path))306 let sem = DispatchSemaphore(value: 0)307 var track: AVAssetTrack?308 Task.detached { track = try? await asset.loadTracks(withMediaType: .video).first; sem.signal() }309 sem.wait()310 guard let track, let reader = try? AVAssetReader(asset: asset) else { return }311 let output = AVAssetReaderTrackOutput(track: track, outputSettings: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA])312 output.alwaysCopiesSampleData = false313 guard reader.canAdd(output) else { return }314 reader.add(output)315 let start = max(0, local.seconds - 0.5)316 reader.timeRange = CMTimeRange(start: CMTime(seconds: start, preferredTimescale: 600), duration: .positiveInfinity)317 guard reader.startReading() else { return }318 current = (index, reader, output, .identity)319 }320}321