SPB Git

Repository Public

VideoExporter.swift 19.7 KB 321 lines swift

1// SP Lecture Studio — LectureRender2// File: Packages/LectureRender/Sources/VideoExporter.swift3// Description: Renders the edited lecture to MP4 (H.264 / HEVC + AAC) with AVAssetWriter: progress, cancellation, disk checks, atomic finish.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import AVFoundation9import CoreGraphics10import CoreVideo11import VideoToolbox12import LectureCore13import LectureStorage14import LecturePDF1516public enum ExportError: Error, Equatable, CustomStringConvertible {17    case nothingToExport18    case cancelled19    case diskFull(needed: Int64, available: Int64)20    case codecUnavailable(String)21    case writerFailed(String)2223    public var description: String {24        switch self {25        case .nothingToExport: "nothing to export"26        case .cancelled: "cancelled"27        case .diskFull(let n, let a): "not enough disk space (needs about \(n / 1_000_000) MB, \(a / 1_000_000) MB available)"28        case .codecUnavailable(let c): "\(c) encoding is not available on this device"29        case .writerFailed(let m): "the video writer failed: \(m)"30        }31    }32}3334public struct ExportResult: Sendable, Hashable {35    public var url: URL36    public var duration: MediaTime37    public var frames: Int38    /// Frames identical to the previous one (the rendered buffer was reused).39    public var reusedFrames: Int40    public var byteCount: Int6441    public var elapsed: TimeInterval42    public var measuredLUFS: Double?43    public var appliedGainDB: Double44    public var unreadableSegments: [String]45}4647/// Everything needed to render a project, loaded once. Shared by the exporter and the editor preview.48public struct RenderSource {49    public var package: ProjectPackage50    public var manifest: ProjectManifest51    public var document: SlideDocument52    public var events: [TimelineEvent]53    public var edits: EditDecisionList54    public var envelope: Envelope?5556    public init(package: ProjectPackage, manifest: ProjectManifest, document: SlideDocument, events: [TimelineEvent], edits: EditDecisionList, envelope: Envelope? = nil) {57        self.package = package; self.manifest = manifest; self.document = document; self.events = events; self.edits = edits; self.envelope = envelope58    }5960    public static func load(_ package: ProjectPackage) throws -> RenderSource {61        let manifest = try package.loadManifest()62        guard manifest.pdf != nil, FileManager.default.fileExists(atPath: package.pdfURL.path) else { throw StorageError.pdfMissing }63        return RenderSource(package: package, manifest: manifest, document: try SlideDocument(url: package.pdfURL),64                            events: package.loadEvents().events, edits: package.loadEdits().current, envelope: package.loadEnvelope())65    }6667    /// Timeline length: the recorded duration, extended to the last event (PDF-only sessions have events but no audio).68    public var duration: MediaTime { manifest.duration }69    public var index: TimelineIndex { TimelineIndex(events: events, adjustments: edits.ink, initialPage: 0) }70    public var timeMap: TimeMap { edits.timeMap(duration: duration) }71}7273public final class VideoExporter {74    public let source: RenderSource75    public let settings: RenderSettings76    public static let audioRate = 48_000.07778    public init(source: RenderSource, settings: RenderSettings) { self.source = source; self.settings = settings }7980    public static func isAvailable(_ codec: VideoCodec) -> Bool {81        switch codec {82        case .h264: return true83        case .hevc: return VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC) || AVAssetExportSession.allExportPresets().contains(AVAssetExportPresetHEVCHighestQuality)84        }85    }8687    /// Exports to `destination`. The file appears there only once complete; a cancelled or failed export leaves nothing behind.88    /// - Parameters:89    ///   - progress: called with 0…1 from background queues.90    ///   - isCancelled: polled between frames.91    public func export(to destination: URL, progress: @escaping @Sendable (Double) -> Void = { _ in },92                       isCancelled: @escaping @Sendable () -> Bool = { false }) async throws -> ExportResult {93        let started = Date()94        let map = source.timeMap95        guard map.outputDuration > .zero else { throw ExportError.nothingToExport }96        guard Self.isAvailable(settings.codec) else { throw ExportError.codecUnavailable(settings.codec.rawValue) }97        let fm = FileManager.default98        let needed = settings.estimatedBytes(duration: map.outputDuration)99        let available = ProjectPackage.freeDiskSpace(at: destination.deletingLastPathComponent())100        if available > 0, available < needed + 50_000_000 { throw ExportError.diskFull(needed: needed, available: available) }101102        // Work in a private scratch folder next to the destination (same volume, so the final move is atomic). AVAssetWriter103        // creates side files of its own there; removing the folder removes every trace of an unfinished export.104        let scratch = destination.deletingLastPathComponent().appendingPathComponent(".export.partial-\(UUID().uuidString.prefix(8))", isDirectory: true)105        try fm.createDirectory(at: scratch, withIntermediateDirectories: true)106        let partial = scratch.appendingPathComponent(destination.lastPathComponent)107        func cleanup() { try? fm.removeItem(at: scratch) }108109        // Audio: optional measurement pass, then the final gain.110        let reader = TimelineAudioReader(package: source.package, manifest: source.manifest, sampleRate: Self.audioRate)111        var processing = AudioProcessing()112        processing.noiseReduction = source.edits.noiseReduction113        if let env = source.envelope { processing.expanderThresholdDB = min(-35, env.percentile(0.03) + 8) }114        let audio = EditedAudioRenderer(reader: reader, map: map, edits: source.edits, processing: processing)115        var measured: (lufs: Double?, peak: Float) = (nil, 0)116        let needsMeasure = source.edits.normalizeLoudness || source.edits.gainDB > 0117        if needsMeasure {118            measured = audio.measure { p in progress(p * 0.08); return !isCancelled() }119            if isCancelled() { throw ExportError.cancelled }120        }121        audio.processing.gain = needsMeasure ? EditedAudioRenderer.resolveGain(edits: source.edits, measured: measured) : pow(10, Float(source.edits.gainDB) / 20)122        audio.reset()123        let base = needsMeasure ? 0.08 : 0.0124125        let writer: AVAssetWriter126        do { writer = try AVAssetWriter(outputURL: partial, fileType: .mp4) } catch { throw ExportError.writerFailed(error.localizedDescription) }127        writer.shouldOptimizeForNetworkUse = true128129        var compression: [String: Any] = [AVVideoAverageBitRateKey: settings.resolvedVideoBitrate, AVVideoExpectedSourceFrameRateKey: settings.fps,130                                          AVVideoMaxKeyFrameIntervalKey: settings.fps * 2]131        if settings.codec == .h264 { compression[AVVideoProfileLevelKey] = AVVideoProfileLevelH264HighAutoLevel }132        let videoSettings: [String: Any] = [133            AVVideoCodecKey: settings.codec == .hevc ? AVVideoCodecType.hevc : AVVideoCodecType.h264,134            AVVideoWidthKey: settings.width, AVVideoHeightKey: settings.height,135            AVVideoColorPropertiesKey: [AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2,136                                        AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2],137            AVVideoCompressionPropertiesKey: compression,138        ]139        let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)140        videoInput.expectsMediaDataInRealTime = false141        let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoInput, sourcePixelBufferAttributes: [142            kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, kCVPixelBufferWidthKey as String: settings.width,143            kCVPixelBufferHeightKey as String: settings.height, kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,144        ])145        let audioSettings: [String: Any] = [AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: Self.audioRate, AVNumberOfChannelsKey: 1,146                                            AVEncoderBitRateKey: min(settings.audioBitrate, 160_000)]147        let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings)148        audioInput.expectsMediaDataInRealTime = false149        guard writer.canAdd(videoInput), writer.canAdd(audioInput) else { cleanup(); throw ExportError.writerFailed("inputs rejected") }150        writer.add(videoInput); writer.add(audioInput)151        guard writer.startWriting() else { cleanup(); throw ExportError.writerFailed(writer.error?.localizedDescription ?? "cannot start") }152        writer.startSession(atSourceTime: .zero)153154        let fps = Int64(settings.fps)155        let totalFrames = Int(max(1, (Double(map.outputDuration.ns) * Double(fps) / 1e9).rounded(.up)))156        let compositor = FrameCompositor(document: source.document, index: source.index, settings: settings)157        let cameraSegments = settings.includeCamera && !source.edits.cameraHidden ? source.manifest.segments(.camera) : []158        let camera = cameraSegments.isEmpty ? nil : CameraFrameProvider(package: source.package, segments: cameraSegments)159        let placementOverride = source.edits.cameraPlacement160161        final class Shared: @unchecked Sendable { var frame = 0; var reused = 0; var audioPos: Int64 = 0; var cancelled = false; var failure: String? }162        let shared = Shared()163        let videoQueue = DispatchQueue(label: "ai.spboucher.lecture.export.video"), audioQueue = DispatchQueue(label: "ai.spboucher.lecture.export.audio")164165        async let videoDone: Void = withCheckedContinuation { (cont: CheckedContinuation<Void, Never>) in166            var lastSignature: FrameSignature?167            var lastBuffer: CVPixelBuffer?168            videoInput.requestMediaDataWhenReady(on: videoQueue) {169                while videoInput.isReadyForMoreMediaData {170                    if shared.frame >= totalFrames || shared.cancelled || shared.failure != nil { videoInput.markAsFinished(); cont.resume(); return }171                    if isCancelled() { shared.cancelled = true; continue }172                    let i = shared.frame173                    let outTime = MediaTime(ns: Int64(i) * 1_000_000_000 / fps)174                    let src = map.toSource(outTime)175                    let sig = compositor.signature(at: src)176                    let cameraImage = camera.flatMap { sig.camera?.enabled ?? true ? $0.frame(at: src) : nil }177                    var buffer: CVPixelBuffer?178                    if cameraImage == nil, sig == lastSignature, let lastBuffer {179                        buffer = lastBuffer; shared.reused += 1180                    } else if let pool = adaptor.pixelBufferPool {181                        var fresh: CVPixelBuffer?182                        CVPixelBufferPoolCreatePixelBuffer(nil, pool, &fresh)183                        if let fresh {184                            CVPixelBufferLockBaseAddress(fresh, [])185                            if let ctx = Bitmap.context(width: CVPixelBufferGetWidth(fresh), height: CVPixelBufferGetHeight(fresh),186                                                        data: CVPixelBufferGetBaseAddress(fresh), bytesPerRow: CVPixelBufferGetBytesPerRow(fresh)) {187                                compositor.render(at: src, into: ctx, camera: cameraImage, cameraPlacement: placementOverride)188                            }189                            CVPixelBufferUnlockBaseAddress(fresh, [])190                            buffer = fresh; lastBuffer = fresh; lastSignature = sig191                        }192                    }193                    guard let buffer, adaptor.append(buffer, withPresentationTime: CMTime(value: Int64(i), timescale: CMTimeScale(fps))) else {194                        shared.failure = writer.error.map { Self.describe($0) } ?? "frame \(i) could not be appended"; continue195                    }196                    shared.frame += 1197                    if i % 15 == 0 { progress(base + (1 - base) * Double(i) / Double(totalFrames) * 0.98) }198                }199            }200        }201202        async let audioDone: Void = withCheckedContinuation { (cont: CheckedContinuation<Void, Never>) in203            let total = audio.totalFrames204            let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.audioRate, channels: 1, interleaved: false)!205            audioInput.requestMediaDataWhenReady(on: audioQueue) {206                while audioInput.isReadyForMoreMediaData {207                    if shared.audioPos >= total || shared.cancelled || shared.failure != nil { audioInput.markAsFinished(); cont.resume(); return }208                    let n = Int(min(Int64(24_000), total - shared.audioPos))209                    let samples = audio.render(start: shared.audioPos, count: n)210                    guard let sb = Self.sampleBuffer(samples, format: format, position: shared.audioPos), audioInput.append(sb) else {211                        shared.failure = writer.error.map { Self.describe($0) } ?? "audio could not be appended"; continue212                    }213                    shared.audioPos += Int64(n)214                }215            }216        }217        _ = await (videoDone, audioDone)218219        if shared.cancelled { writer.cancelWriting(); cleanup(); throw ExportError.cancelled }220        if let failure = shared.failure {221            let err = writer.error222            writer.cancelWriting(); cleanup()223            if let err, Self.isDiskFull(err) { throw ExportError.diskFull(needed: needed, available: ProjectPackage.freeDiskSpace(at: destination.deletingLastPathComponent())) }224            throw ExportError.writerFailed(failure)225        }226        writer.endSession(atSourceTime: CMTime(value: map.outputDuration.ns, timescale: MediaTime.timescale))227        await writer.finishWriting()228        guard writer.status == .completed else {229            let err = writer.error; cleanup()230            if let err, Self.isDiskFull(err) { throw ExportError.diskFull(needed: needed, available: 0) }231            throw ExportError.writerFailed(err.map { Self.describe($0) } ?? "unknown")232        }233        // Atomic hand-over: the destination is replaced only by a complete file.234        if fm.fileExists(atPath: destination.path) { _ = try? fm.replaceItemAt(destination, withItemAt: partial) } else { try fm.moveItem(at: partial, to: destination) }235        cleanup()236        progress(1)237        let size = (try? fm.attributesOfItem(atPath: destination.path)[.size] as? NSNumber)?.int64Value ?? 0238        return ExportResult(url: destination, duration: map.outputDuration, frames: totalFrames, reusedFrames: shared.reused, byteCount: size,239                            elapsed: Date().timeIntervalSince(started), measuredLUFS: measured.lufs,240                            appliedGainDB: Double(20 * log10(max(audio.processing.gain, 1e-6))), unreadableSegments: reader.unreadable)241    }242243    static func describe(_ error: Error) -> String { let e = error as NSError; return "\(e.domain) \(e.code): \(e.localizedDescription)" }244245    static func isDiskFull(_ error: Error) -> Bool {246        let e = error as NSError247        if e.domain == AVFoundationErrorDomain && e.code == AVError.diskFull.rawValue { return true }248        if e.domain == NSPOSIXErrorDomain && e.code == Int(ENOSPC) { return true }249        if e.domain == NSCocoaErrorDomain && e.code == NSFileWriteOutOfSpaceError { return true }250        if let u = e.userInfo[NSUnderlyingErrorKey] as? NSError { return isDiskFull(u) }251        return false252    }253254    static func sampleBuffer(_ samples: [Float], format: AVAudioFormat, position: Int64) -> CMSampleBuffer? {255        guard let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)) else { return nil }256        pcm.frameLength = AVAudioFrameCount(samples.count)257        samples.withUnsafeBufferPointer { pcm.floatChannelData![0].update(from: $0.baseAddress!, count: samples.count) }258        var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: CMTimeScale(format.sampleRate)),259                                        presentationTimeStamp: CMTime(value: position, timescale: CMTimeScale(format.sampleRate)), decodeTimeStamp: .invalid)260        var sb: CMSampleBuffer?261        guard CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, dataReady: false, makeDataReadyCallback: nil, refcon: nil,262                                   formatDescription: format.formatDescription, sampleCount: CMItemCount(samples.count), sampleTimingEntryCount: 1,263                                   sampleTimingArray: &timing, sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sb) == noErr, let sb else { return nil }264        guard CMSampleBufferSetDataBufferFromAudioBufferList(sb, blockBufferAllocator: kCFAllocatorDefault, blockBufferMemoryAllocator: kCFAllocatorDefault,265                                                             flags: 0, bufferList: pcm.audioBufferList) == noErr else { return nil }266        return sb267    }268}269270/// Sequential reader of the camera segments: returns the camera frame shown at a timeline time.271public final class CameraFrameProvider {272    private let package: ProjectPackage273    private let segments: [MediaSegment]274    private var current: (index: Int, reader: AVAssetReader, output: AVAssetReaderTrackOutput, transform: CGAffineTransform)?275    private var held: (time: MediaTime, image: CGImage)?276    private var pending: (time: MediaTime, image: CGImage)?277278    public init(package: ProjectPackage, segments: [MediaSegment]) {279        self.package = package; self.segments = segments.sorted { $0.start < $1.start }280    }281282    public func frame(at t: MediaTime) -> CGImage? {283        guard let si = segments.firstIndex(where: { $0.range.contains(t) }) else { return nil }284        let seg = segments[si]285        let local = t - seg.start286        // Reopen when the segment changes or when time jumps backwards / far ahead (a cut).287        let needsOpen = current?.index != si || (held.map { $0.time > local } ?? false) || (held.map { (local - $0.time).seconds > 2 } ?? false && pending == nil)288        if needsOpen { open(si, from: local) }289        guard let cur = current else { return nil }290        while true {291            if let p = pending {292                if p.time <= local { held = p; pending = nil } else { break }293            }294            guard cur.reader.status == .reading, let sb = cur.output.copyNextSampleBuffer() else { break }295            guard let pb = CMSampleBufferGetImageBuffer(sb) else { continue }296            var img: CGImage?297            VTCreateCGImageFromCVPixelBuffer(pb, options: nil, imageOut: &img)298            if let img { pending = (MediaTime(seconds: CMSampleBufferGetPresentationTimeStamp(sb).seconds), img) }299        }300        return held?.image301    }302303    private func open(_ index: Int, from local: MediaTime) {304        current?.reader.cancelReading(); current = nil; held = nil; pending = nil305        let asset = AVURLAsset(url: package.url(segments[index].path))306        let sem = DispatchSemaphore(value: 0)307        var track: AVAssetTrack?308        Task.detached { track = try? await asset.loadTracks(withMediaType: .video).first; sem.signal() }309        sem.wait()310        guard let track, let reader = try? AVAssetReader(asset: asset) else { return }311        let output = AVAssetReaderTrackOutput(track: track, outputSettings: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA])312        output.alwaysCopiesSampleData = false313        guard reader.canAdd(output) else { return }314        reader.add(output)315        let start = max(0, local.seconds - 0.5)316        reader.timeRange = CMTimeRange(start: CMTime(seconds: start, preferredTimescale: 600), duration: .positiveInfinity)317        guard reader.startReading() else { return }318        current = (index, reader, output, .identity)319    }320}321
Files main
Menu

Theme