SPB Git

Repository Public

AudioTimeline.swift 16.1 KB 310 lines swift

1// SP Lecture Studio — LectureRender2// File: Packages/LectureRender/Sources/AudioTimeline.swift3// Description: Random-access reader of the segmented source audio, and the edited-audio renderer (cuts, crossfades, mutes, fades, gain, cleanup).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import AVFoundation9import LectureCore10import LectureStorage1112/// Reads the recorded audio of a project as one continuous mono stream at `sampleRate`, whatever the segment boundaries13/// and native sample rates. Missing or unreadable segments read as silence (and are reported in `unreadable`).14public final class TimelineAudioReader {15    public let sampleRate: Double16    public let totalFrames: Int6417    public private(set) var unreadable: [String] = []1819    private struct Entry { var segment: MediaSegment; var url: URL; var startFrame: Int64; var frames: Int64 }20    private var entries: [Entry] = []21    private var cache: [(index: Int, samples: [Float])] = []22    private let cacheLimit = 32324    public init(package: ProjectPackage, manifest: ProjectManifest, sampleRate: Double = 48_000) {25        self.sampleRate = sampleRate26        for seg in manifest.segments(.audio) {27            let start = seg.start.frames(atSampleRate: sampleRate)28            let frames = seg.sampleRate == sampleRate ? seg.frames : seg.duration.frames(atSampleRate: sampleRate)29            entries.append(Entry(segment: seg, url: package.url(seg.path), startFrame: start, frames: frames))30        }31        totalFrames = max(manifest.duration.frames(atSampleRate: sampleRate), entries.map { $0.startFrame + $0.frames }.max() ?? 0)32    }3334    public var hasAudio: Bool { !entries.isEmpty }3536    /// `count` frames starting at `start` (timeline frames at `sampleRate`). Positions without audio are zero.37    public func read(start: Int64, count: Int) -> [Float] {38        var out = [Float](repeating: 0, count: max(0, count))39        guard count > 0 else { return out }40        let end = start + Int64(count)41        for (i, e) in entries.enumerated() where e.startFrame < end && e.startFrame + e.frames > start {42            guard let samples = decoded(i) else { continue }43            let from = max(start, e.startFrame), to = min(end, e.startFrame + Int64(samples.count))44            guard to > from else { continue }45            let src = Int(from - e.startFrame), dst = Int(from - start), n = Int(to - from)46            out.withUnsafeMutableBufferPointer { o in samples.withUnsafeBufferPointer { s in47                o.baseAddress!.advanced(by: dst).update(from: s.baseAddress!.advanced(by: src), count: n)48            } }49        }50        return out51    }5253    private func decoded(_ i: Int) -> [Float]? {54        if let hit = cache.firstIndex(where: { $0.index == i }) {55            let c = cache.remove(at: hit); cache.append(c); return c.samples56        }57        guard let samples = Self.decode(entries[i].url, to: sampleRate) else {58            if !unreadable.contains(entries[i].segment.path) { unreadable.append(entries[i].segment.path) }59            return nil60        }61        cache.append((i, samples)); if cache.count > cacheLimit { cache.removeFirst() }62        return samples63    }6465    /// Decodes a whole audio file to mono float at `rate` (channels averaged, sample-rate converted when needed).66    public static func decode(_ url: URL, to rate: Double) -> [Float]? {67        guard let file = try? AVAudioFile(forReading: url), file.length > 0 else { return nil }68        let inFormat = file.processingFormat69        guard let inBuffer = AVAudioPCMBuffer(pcmFormat: inFormat, frameCapacity: AVAudioFrameCount(file.length)) else { return nil }70        do { try file.read(into: inBuffer) } catch { if inBuffer.frameLength == 0 { return nil } }71        let n = Int(inBuffer.frameLength), ch = Int(inFormat.channelCount)72        var mono = [Float](repeating: 0, count: n)73        guard let data = inBuffer.floatChannelData else { return nil }74        for c in 0..<ch { let p = data[c]; for i in 0..<n { mono[i] += p[i] } }75        if ch > 1 { let k = 1 / Float(ch); for i in 0..<n { mono[i] *= k } }76        guard inFormat.sampleRate != rate else { return mono }77        guard let srcFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: inFormat.sampleRate, channels: 1, interleaved: false),78              let dstFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: rate, channels: 1, interleaved: false),79              let converter = AVAudioConverter(from: srcFormat, to: dstFormat),80              let src = AVAudioPCMBuffer(pcmFormat: srcFormat, frameCapacity: AVAudioFrameCount(n)) else { return nil }81        src.frameLength = AVAudioFrameCount(n)82        mono.withUnsafeBufferPointer { src.floatChannelData![0].update(from: $0.baseAddress!, count: n) }83        let outCapacity = AVAudioFrameCount(Double(n) * rate / inFormat.sampleRate + 64)84        guard let dst = AVAudioPCMBuffer(pcmFormat: dstFormat, frameCapacity: outCapacity) else { return nil }85        var fed = false86        var error: NSError?87        converter.convert(to: dst, error: &error) { _, status in88            if fed { status.pointee = .endOfStream; return nil }89            fed = true; status.pointee = .haveData; return src90        }91        guard error == nil else { return nil }92        return Array(UnsafeBufferPointer(start: dst.floatChannelData![0], count: Int(dst.frameLength)))93    }9495    /// Loudness envelope of the whole recording (used for the waveform, silence detection and the noise floor).96    public func envelope(hop: MediaTime = MediaTime(milliseconds: 10), progress: ((Double) -> Bool)? = nil) -> Envelope {97        var env = Envelope(hop: hop)98        let hopFrames = Int(hop.frames(atSampleRate: sampleRate))99        let block = hopFrames * 3000            // 30 s100        var pos: Int64 = 0101        while pos < totalFrames {102            let n = Int(min(Int64(block), totalFrames - pos))103            let samples = read(start: pos, count: n - n % hopFrames == 0 ? n : n - n % hopFrames)104            if samples.isEmpty { break }105            samples.withUnsafeBufferPointer { env.append(Envelope.compute(samples: $0, sampleRate: sampleRate, hop: hop)) }106            pos += Int64(samples.count)107            if let progress, !progress(Double(pos) / Double(max(1, totalFrames))) { break }108        }109        return env110    }111}112113/// Second-order IIR section (transposed direct form II).114struct Biquad {115    var b0: Float, b1: Float, b2: Float, a1: Float, a2: Float116    var z1: Float = 0, z2: Float = 0117    mutating func process(_ x: Float) -> Float {118        let y = b0 * x + z1119        z1 = b1 * x - a1 * y + z2120        z2 = b2 * x - a2 * y121        return y122    }123    mutating func reset() { z1 = 0; z2 = 0 }124    static func highPass(hz: Double, q: Double = 0.707, sampleRate: Double) -> Biquad {125        let w = 2 * Double.pi * hz / sampleRate, alpha = sin(w) / (2 * q), c = cos(w), a0 = 1 + alpha126        return Biquad(b0: Float((1 + c) / 2 / a0), b1: Float(-(1 + c) / a0), b2: Float((1 + c) / 2 / a0), a1: Float(-2 * c / a0), a2: Float((1 - alpha) / a0))127    }128}129130/// Integrated loudness per ITU-R BS.1770 (K-weighting, 400 ms blocks with 75 % overlap, absolute and relative gating), mono, 48 kHz.131public struct LoudnessMeter {132    private var shelf = Biquad(b0: 1.53512485958697, b1: -2.69169618940638, b2: 1.19839281085285, a1: -1.69065929318241, a2: 0.73248077421585)133    private var highPass = Biquad(b0: 1.0, b1: -2.0, b2: 1.0, a1: -1.99004745483398, a2: 0.99007225036621)134    private var stepEnergies: [Double] = []          // energy of each 100 ms step135    private var acc: Double = 0, accCount = 0136    private let step = 4800137    public private(set) var peak: Float = 0138    public init() {}139140    public mutating func process(_ samples: [Float]) {141        for x in samples {142            peak = max(peak, abs(x))143            let y = Double(highPass.process(shelf.process(x)))144            acc += y * y; accCount += 1145            if accCount == step { stepEnergies.append(acc / Double(step)); acc = 0; accCount = 0 }146        }147    }148149    /// Integrated loudness in LUFS, or nil when the programme is shorter than one block or entirely below the absolute gate.150    public func integratedLUFS() -> Double? {151        guard stepEnergies.count >= 4 else { return nil }152        var blocks: [Double] = []153        for i in 0...(stepEnergies.count - 4) { blocks.append((stepEnergies[i] + stepEnergies[i + 1] + stepEnergies[i + 2] + stepEnergies[i + 3]) / 4) }154        func lufs(_ ms: Double) -> Double { -0.691 + 10 * log10(max(ms, 1e-12)) }155        let abs = blocks.filter { lufs($0) > -70 }156        guard !abs.isEmpty else { return nil }157        let relThreshold = lufs(abs.reduce(0, +) / Double(abs.count)) - 10158        let rel = abs.filter { lufs($0) > relThreshold }159        guard !rel.isEmpty else { return nil }160        return lufs(rel.reduce(0, +) / Double(rel.count))161    }162}163164public struct AudioProcessing: Sendable, Hashable {165    /// Linear gain applied to the whole programme (manual gain × loudness normalization).166    public var gain: Float = 1167    /// Conservative cleanup: 80 Hz high-pass (rumble) + gentle downward expander below `expanderThresholdDB`.168    public var noiseReduction = false169    public var expanderThresholdDB: Float = -50170    public init() {}171}172173/// Produces the audio of the edited timeline. The same renderer feeds the preview player and the exporter,174/// which is what guarantees that what is heard while editing is what gets exported.175public final class EditedAudioRenderer {176    public static let targetLUFS = -16.0177    public let reader: TimelineAudioReader178    public let map: TimeMap179    public let edits: EditDecisionList180    public var processing: AudioProcessing181    public let sampleRate: Double182    public let totalFrames: Int64183184    private struct Kept { var srcStart: Int64; var srcEnd: Int64; var outStart: Int64 }185    private var kept: [Kept] = []186    private var mutes: [(Int64, Int64)] = []          // source frames187    private var hp: Biquad188    private var expanderEnv: Float = 0, expanderGain: Float = 1189    private var nextSequentialFrame: Int64 = -1190191    public init(reader: TimelineAudioReader, map: TimeMap, edits: EditDecisionList, processing: AudioProcessing = AudioProcessing()) {192        self.reader = reader; self.map = map; self.edits = edits; self.processing = processing193        let rate = reader.sampleRate194        sampleRate = rate195        var out: Int64 = 0196        var list: [Kept] = []197        for k in map.kept {198            let s = k.start.frames(atSampleRate: rate), e = k.end.frames(atSampleRate: rate)199            guard e > s else { continue }200            list.append(Kept(srcStart: s, srcEnd: e, outStart: out)); out += e - s201        }202        kept = list203        totalFrames = out204        mutes = TimeRange.merged(edits.muted).map { ($0.start.frames(atSampleRate: rate), $0.end.frames(atSampleRate: rate)) }205        hp = Biquad.highPass(hz: 80, sampleRate: rate)206    }207208    public func reset() { hp.reset(); expanderEnv = 0; expanderGain = 1; nextSequentialFrame = -1 }209210    private func source(_ start: Int64, _ count: Int) -> [Float] {211        var s = reader.read(start: start, count: count)212        for (a, b) in mutes where a < start + Int64(count) && b > start {213            let lo = Int(max(a, start) - start), hi = Int(min(b, start + Int64(count)) - start)214            for i in lo..<hi { s[i] = 0 }215        }216        return s217    }218219    /// Renders `count` frames of the output timeline starting at `start` (zero-padded past the end).220    public func render(start: Int64, count: Int) -> [Float] {221        var out = [Float](repeating: 0, count: max(0, count))222        guard count > 0 else { return out }223        if start != nextSequentialFrame { hp.reset(); expanderEnv = 0; expanderGain = 1 }224        nextSequentialFrame = start + Int64(count)225        let end = start + Int64(count)226227        // 1. Kept ranges, copied straight from the source.228        for k in kept {229            let kOutEnd = k.outStart + (k.srcEnd - k.srcStart)230            guard k.outStart < end, kOutEnd > start else { continue }231            let from = max(start, k.outStart), to = min(end, kOutEnd)232            let piece = source(k.srcStart + (from - k.outStart), Int(to - from))233            out.replaceSubrange(Int(from - start)..<Int(to - start), with: piece)234        }235236        // 2. Equal-power crossfade centred on every join, using handle material from both sides of the cut.237        let half = Int64((edits.crossfade.seconds * sampleRate / 2).rounded())238        if half > 0, kept.count > 1 {239            for j in 1..<kept.count {240                let a = kept[j - 1], b = kept[j], join = b.outStart241                let h = min(half, (a.srcEnd - a.srcStart) / 2, (b.srcEnd - b.srcStart) / 2, max(0, b.srcStart - a.srcEnd))242                guard h > 0, join - h < end, join + h > start else { continue }243                let from = max(start, join - h), to = min(end, join + h), n = Int(to - from)244                let outgoing = source(a.srcEnd + (from - join), n)        // continues past the cut point into the removed material245                let incoming = source(b.srcStart + (from - join), n)      // starts inside the removed material before the kept range246                for i in 0..<n {247                    let x = Float(Double(from + Int64(i) - (join - h)) / Double(2 * h))248                    out[Int(from - start) + i] = outgoing[i] * cos(x * .pi / 2) + incoming[i] * sin(x * .pi / 2)249                }250            }251        }252253        // 3. Cleanup, gain, fades.254        if processing.noiseReduction {255            let thr = processing.expanderThresholdDB256            let attack = Float(exp(-1 / (0.005 * sampleRate))), release = Float(exp(-1 / (0.200 * sampleRate)))257            for i in 0..<count {258                let y = hp.process(out[i])259                let level = abs(y)260                expanderEnv = level > expanderEnv ? attack * expanderEnv + (1 - attack) * level : release * expanderEnv + (1 - release) * level261                let db = Envelope.dB(expanderEnv)262                // 2:1 below the threshold, limited to 12 dB of attenuation, smoothed to avoid pumping.263                let target: Float = db < thr ? pow(10, max(-12, (db - thr)) / 20) : 1264                expanderGain += (target - expanderGain) * 0.002265                out[i] = y * expanderGain266            }267        }268        let g = processing.gain269        let fadeIn = Int64(edits.fadeIn.seconds * sampleRate), fadeOut = Int64(edits.fadeOut.seconds * sampleRate)270        for i in 0..<count {271            let pos = start + Int64(i)272            var gain = g273            if pos < fadeIn { gain *= Float(pos) / Float(fadeIn) }274            if fadeOut > 0, pos >= totalFrames - fadeOut { gain *= max(0, Float(totalFrames - pos) / Float(fadeOut)) }275            if pos >= totalFrames { gain = 0 }276            out[i] = min(1, max(-1, out[i] * gain))277        }278        return out279    }280281    /// One pass over the edited programme (with cleanup, without gain) to measure loudness and peak.282    public func measure(progress: ((Double) -> Bool)? = nil) -> (lufs: Double?, peak: Float) {283        let saved = processing284        processing.gain = 1285        defer { processing = saved; reset() }286        reset()287        var meter = LoudnessMeter()288        var pos: Int64 = 0289        let block = 48_000 * 10290        // Fades are part of the programme but should not bias the measurement much; they are left in.291        while pos < totalFrames {292            let n = Int(min(Int64(block), totalFrames - pos))293            meter.process(render(start: pos, count: n)); pos += Int64(n)294            if let progress, !progress(Double(pos) / Double(max(1, totalFrames))) { break }295        }296        return (meter.integratedLUFS(), meter.peak)297    }298299    /// Final linear gain: manual gain, plus (when enabled) normalization to -16 LUFS, capped at +20 dB and at a -1 dBFS peak.300    public static func resolveGain(edits: EditDecisionList, measured: (lufs: Double?, peak: Float)) -> Float {301        var gain = pow(10, Float(edits.gainDB) / 20)302        if edits.normalizeLoudness, let lufs = measured.lufs {303            gain *= Float(pow(10, min(20, targetLUFS - lufs) / 20))304        }305        // Amplification never pushes the peak above -1 dBFS; a recording that needs no boost is left untouched.306        if gain > 1, measured.peak > 0 { gain = max(1, min(gain, 0.891 / measured.peak)) }307        return gain308    }309}310
Files main
Menu

Theme