Repository Public
1// SP Lecture Studio — LectureRender2// File: Packages/LectureRender/Sources/AudioTimeline.swift3// Description: Random-access reader of the segmented source audio, and the edited-audio renderer (cuts, crossfades, mutes, fades, gain, cleanup).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import AVFoundation9import LectureCore10import LectureStorage1112/// Reads the recorded audio of a project as one continuous mono stream at `sampleRate`, whatever the segment boundaries13/// and native sample rates. Missing or unreadable segments read as silence (and are reported in `unreadable`).14public final class TimelineAudioReader {15 public let sampleRate: Double16 public let totalFrames: Int6417 public private(set) var unreadable: [String] = []1819 private struct Entry { var segment: MediaSegment; var url: URL; var startFrame: Int64; var frames: Int64 }20 private var entries: [Entry] = []21 private var cache: [(index: Int, samples: [Float])] = []22 private let cacheLimit = 32324 public init(package: ProjectPackage, manifest: ProjectManifest, sampleRate: Double = 48_000) {25 self.sampleRate = sampleRate26 for seg in manifest.segments(.audio) {27 let start = seg.start.frames(atSampleRate: sampleRate)28 let frames = seg.sampleRate == sampleRate ? seg.frames : seg.duration.frames(atSampleRate: sampleRate)29 entries.append(Entry(segment: seg, url: package.url(seg.path), startFrame: start, frames: frames))30 }31 totalFrames = max(manifest.duration.frames(atSampleRate: sampleRate), entries.map { $0.startFrame + $0.frames }.max() ?? 0)32 }3334 public var hasAudio: Bool { !entries.isEmpty }3536 /// `count` frames starting at `start` (timeline frames at `sampleRate`). Positions without audio are zero.37 public func read(start: Int64, count: Int) -> [Float] {38 var out = [Float](repeating: 0, count: max(0, count))39 guard count > 0 else { return out }40 let end = start + Int64(count)41 for (i, e) in entries.enumerated() where e.startFrame < end && e.startFrame + e.frames > start {42 guard let samples = decoded(i) else { continue }43 let from = max(start, e.startFrame), to = min(end, e.startFrame + Int64(samples.count))44 guard to > from else { continue }45 let src = Int(from - e.startFrame), dst = Int(from - start), n = Int(to - from)46 out.withUnsafeMutableBufferPointer { o in samples.withUnsafeBufferPointer { s in47 o.baseAddress!.advanced(by: dst).update(from: s.baseAddress!.advanced(by: src), count: n)48 } }49 }50 return out51 }5253 private func decoded(_ i: Int) -> [Float]? {54 if let hit = cache.firstIndex(where: { $0.index == i }) {55 let c = cache.remove(at: hit); cache.append(c); return c.samples56 }57 guard let samples = Self.decode(entries[i].url, to: sampleRate) else {58 if !unreadable.contains(entries[i].segment.path) { unreadable.append(entries[i].segment.path) }59 return nil60 }61 cache.append((i, samples)); if cache.count > cacheLimit { cache.removeFirst() }62 return samples63 }6465 /// Decodes a whole audio file to mono float at `rate` (channels averaged, sample-rate converted when needed).66 public static func decode(_ url: URL, to rate: Double) -> [Float]? {67 guard let file = try? AVAudioFile(forReading: url), file.length > 0 else { return nil }68 let inFormat = file.processingFormat69 guard let inBuffer = AVAudioPCMBuffer(pcmFormat: inFormat, frameCapacity: AVAudioFrameCount(file.length)) else { return nil }70 do { try file.read(into: inBuffer) } catch { if inBuffer.frameLength == 0 { return nil } }71 let n = Int(inBuffer.frameLength), ch = Int(inFormat.channelCount)72 var mono = [Float](repeating: 0, count: n)73 guard let data = inBuffer.floatChannelData else { return nil }74 for c in 0..<ch { let p = data[c]; for i in 0..<n { mono[i] += p[i] } }75 if ch > 1 { let k = 1 / Float(ch); for i in 0..<n { mono[i] *= k } }76 guard inFormat.sampleRate != rate else { return mono }77 guard let srcFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: inFormat.sampleRate, channels: 1, interleaved: false),78 let dstFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: rate, channels: 1, interleaved: false),79 let converter = AVAudioConverter(from: srcFormat, to: dstFormat),80 let src = AVAudioPCMBuffer(pcmFormat: srcFormat, frameCapacity: AVAudioFrameCount(n)) else { return nil }81 src.frameLength = AVAudioFrameCount(n)82 mono.withUnsafeBufferPointer { src.floatChannelData![0].update(from: $0.baseAddress!, count: n) }83 let outCapacity = AVAudioFrameCount(Double(n) * rate / inFormat.sampleRate + 64)84 guard let dst = AVAudioPCMBuffer(pcmFormat: dstFormat, frameCapacity: outCapacity) else { return nil }85 var fed = false86 var error: NSError?87 converter.convert(to: dst, error: &error) { _, status in88 if fed { status.pointee = .endOfStream; return nil }89 fed = true; status.pointee = .haveData; return src90 }91 guard error == nil else { return nil }92 return Array(UnsafeBufferPointer(start: dst.floatChannelData![0], count: Int(dst.frameLength)))93 }9495 /// Loudness envelope of the whole recording (used for the waveform, silence detection and the noise floor).96 public func envelope(hop: MediaTime = MediaTime(milliseconds: 10), progress: ((Double) -> Bool)? = nil) -> Envelope {97 var env = Envelope(hop: hop)98 let hopFrames = Int(hop.frames(atSampleRate: sampleRate))99 let block = hopFrames * 3000 // 30 s100 var pos: Int64 = 0101 while pos < totalFrames {102 let n = Int(min(Int64(block), totalFrames - pos))103 let samples = read(start: pos, count: n - n % hopFrames == 0 ? n : n - n % hopFrames)104 if samples.isEmpty { break }105 samples.withUnsafeBufferPointer { env.append(Envelope.compute(samples: $0, sampleRate: sampleRate, hop: hop)) }106 pos += Int64(samples.count)107 if let progress, !progress(Double(pos) / Double(max(1, totalFrames))) { break }108 }109 return env110 }111}112113/// Second-order IIR section (transposed direct form II).114struct Biquad {115 var b0: Float, b1: Float, b2: Float, a1: Float, a2: Float116 var z1: Float = 0, z2: Float = 0117 mutating func process(_ x: Float) -> Float {118 let y = b0 * x + z1119 z1 = b1 * x - a1 * y + z2120 z2 = b2 * x - a2 * y121 return y122 }123 mutating func reset() { z1 = 0; z2 = 0 }124 static func highPass(hz: Double, q: Double = 0.707, sampleRate: Double) -> Biquad {125 let w = 2 * Double.pi * hz / sampleRate, alpha = sin(w) / (2 * q), c = cos(w), a0 = 1 + alpha126 return Biquad(b0: Float((1 + c) / 2 / a0), b1: Float(-(1 + c) / a0), b2: Float((1 + c) / 2 / a0), a1: Float(-2 * c / a0), a2: Float((1 - alpha) / a0))127 }128}129130/// Integrated loudness per ITU-R BS.1770 (K-weighting, 400 ms blocks with 75 % overlap, absolute and relative gating), mono, 48 kHz.131public struct LoudnessMeter {132 private var shelf = Biquad(b0: 1.53512485958697, b1: -2.69169618940638, b2: 1.19839281085285, a1: -1.69065929318241, a2: 0.73248077421585)133 private var highPass = Biquad(b0: 1.0, b1: -2.0, b2: 1.0, a1: -1.99004745483398, a2: 0.99007225036621)134 private var stepEnergies: [Double] = [] // energy of each 100 ms step135 private var acc: Double = 0, accCount = 0136 private let step = 4800137 public private(set) var peak: Float = 0138 public init() {}139140 public mutating func process(_ samples: [Float]) {141 for x in samples {142 peak = max(peak, abs(x))143 let y = Double(highPass.process(shelf.process(x)))144 acc += y * y; accCount += 1145 if accCount == step { stepEnergies.append(acc / Double(step)); acc = 0; accCount = 0 }146 }147 }148149 /// Integrated loudness in LUFS, or nil when the programme is shorter than one block or entirely below the absolute gate.150 public func integratedLUFS() -> Double? {151 guard stepEnergies.count >= 4 else { return nil }152 var blocks: [Double] = []153 for i in 0...(stepEnergies.count - 4) { blocks.append((stepEnergies[i] + stepEnergies[i + 1] + stepEnergies[i + 2] + stepEnergies[i + 3]) / 4) }154 func lufs(_ ms: Double) -> Double { -0.691 + 10 * log10(max(ms, 1e-12)) }155 let abs = blocks.filter { lufs($0) > -70 }156 guard !abs.isEmpty else { return nil }157 let relThreshold = lufs(abs.reduce(0, +) / Double(abs.count)) - 10158 let rel = abs.filter { lufs($0) > relThreshold }159 guard !rel.isEmpty else { return nil }160 return lufs(rel.reduce(0, +) / Double(rel.count))161 }162}163164public struct AudioProcessing: Sendable, Hashable {165 /// Linear gain applied to the whole programme (manual gain × loudness normalization).166 public var gain: Float = 1167 /// Conservative cleanup: 80 Hz high-pass (rumble) + gentle downward expander below `expanderThresholdDB`.168 public var noiseReduction = false169 public var expanderThresholdDB: Float = -50170 public init() {}171}172173/// Produces the audio of the edited timeline. The same renderer feeds the preview player and the exporter,174/// which is what guarantees that what is heard while editing is what gets exported.175public final class EditedAudioRenderer {176 public static let targetLUFS = -16.0177 public let reader: TimelineAudioReader178 public let map: TimeMap179 public let edits: EditDecisionList180 public var processing: AudioProcessing181 public let sampleRate: Double182 public let totalFrames: Int64183184 private struct Kept { var srcStart: Int64; var srcEnd: Int64; var outStart: Int64 }185 private var kept: [Kept] = []186 private var mutes: [(Int64, Int64)] = [] // source frames187 private var hp: Biquad188 private var expanderEnv: Float = 0, expanderGain: Float = 1189 private var nextSequentialFrame: Int64 = -1190191 public init(reader: TimelineAudioReader, map: TimeMap, edits: EditDecisionList, processing: AudioProcessing = AudioProcessing()) {192 self.reader = reader; self.map = map; self.edits = edits; self.processing = processing193 let rate = reader.sampleRate194 sampleRate = rate195 var out: Int64 = 0196 var list: [Kept] = []197 for k in map.kept {198 let s = k.start.frames(atSampleRate: rate), e = k.end.frames(atSampleRate: rate)199 guard e > s else { continue }200 list.append(Kept(srcStart: s, srcEnd: e, outStart: out)); out += e - s201 }202 kept = list203 totalFrames = out204 mutes = TimeRange.merged(edits.muted).map { ($0.start.frames(atSampleRate: rate), $0.end.frames(atSampleRate: rate)) }205 hp = Biquad.highPass(hz: 80, sampleRate: rate)206 }207208 public func reset() { hp.reset(); expanderEnv = 0; expanderGain = 1; nextSequentialFrame = -1 }209210 private func source(_ start: Int64, _ count: Int) -> [Float] {211 var s = reader.read(start: start, count: count)212 for (a, b) in mutes where a < start + Int64(count) && b > start {213 let lo = Int(max(a, start) - start), hi = Int(min(b, start + Int64(count)) - start)214 for i in lo..<hi { s[i] = 0 }215 }216 return s217 }218219 /// Renders `count` frames of the output timeline starting at `start` (zero-padded past the end).220 public func render(start: Int64, count: Int) -> [Float] {221 var out = [Float](repeating: 0, count: max(0, count))222 guard count > 0 else { return out }223 if start != nextSequentialFrame { hp.reset(); expanderEnv = 0; expanderGain = 1 }224 nextSequentialFrame = start + Int64(count)225 let end = start + Int64(count)226227 // 1. Kept ranges, copied straight from the source.228 for k in kept {229 let kOutEnd = k.outStart + (k.srcEnd - k.srcStart)230 guard k.outStart < end, kOutEnd > start else { continue }231 let from = max(start, k.outStart), to = min(end, kOutEnd)232 let piece = source(k.srcStart + (from - k.outStart), Int(to - from))233 out.replaceSubrange(Int(from - start)..<Int(to - start), with: piece)234 }235236 // 2. Equal-power crossfade centred on every join, using handle material from both sides of the cut.237 let half = Int64((edits.crossfade.seconds * sampleRate / 2).rounded())238 if half > 0, kept.count > 1 {239 for j in 1..<kept.count {240 let a = kept[j - 1], b = kept[j], join = b.outStart241 let h = min(half, (a.srcEnd - a.srcStart) / 2, (b.srcEnd - b.srcStart) / 2, max(0, b.srcStart - a.srcEnd))242 guard h > 0, join - h < end, join + h > start else { continue }243 let from = max(start, join - h), to = min(end, join + h), n = Int(to - from)244 let outgoing = source(a.srcEnd + (from - join), n) // continues past the cut point into the removed material245 let incoming = source(b.srcStart + (from - join), n) // starts inside the removed material before the kept range246 for i in 0..<n {247 let x = Float(Double(from + Int64(i) - (join - h)) / Double(2 * h))248 out[Int(from - start) + i] = outgoing[i] * cos(x * .pi / 2) + incoming[i] * sin(x * .pi / 2)249 }250 }251 }252253 // 3. Cleanup, gain, fades.254 if processing.noiseReduction {255 let thr = processing.expanderThresholdDB256 let attack = Float(exp(-1 / (0.005 * sampleRate))), release = Float(exp(-1 / (0.200 * sampleRate)))257 for i in 0..<count {258 let y = hp.process(out[i])259 let level = abs(y)260 expanderEnv = level > expanderEnv ? attack * expanderEnv + (1 - attack) * level : release * expanderEnv + (1 - release) * level261 let db = Envelope.dB(expanderEnv)262 // 2:1 below the threshold, limited to 12 dB of attenuation, smoothed to avoid pumping.263 let target: Float = db < thr ? pow(10, max(-12, (db - thr)) / 20) : 1264 expanderGain += (target - expanderGain) * 0.002265 out[i] = y * expanderGain266 }267 }268 let g = processing.gain269 let fadeIn = Int64(edits.fadeIn.seconds * sampleRate), fadeOut = Int64(edits.fadeOut.seconds * sampleRate)270 for i in 0..<count {271 let pos = start + Int64(i)272 var gain = g273 if pos < fadeIn { gain *= Float(pos) / Float(fadeIn) }274 if fadeOut > 0, pos >= totalFrames - fadeOut { gain *= max(0, Float(totalFrames - pos) / Float(fadeOut)) }275 if pos >= totalFrames { gain = 0 }276 out[i] = min(1, max(-1, out[i] * gain))277 }278 return out279 }280281 /// One pass over the edited programme (with cleanup, without gain) to measure loudness and peak.282 public func measure(progress: ((Double) -> Bool)? = nil) -> (lufs: Double?, peak: Float) {283 let saved = processing284 processing.gain = 1285 defer { processing = saved; reset() }286 reset()287 var meter = LoudnessMeter()288 var pos: Int64 = 0289 let block = 48_000 * 10290 // Fades are part of the programme but should not bias the measurement much; they are left in.291 while pos < totalFrames {292 let n = Int(min(Int64(block), totalFrames - pos))293 meter.process(render(start: pos, count: n)); pos += Int64(n)294 if let progress, !progress(Double(pos) / Double(max(1, totalFrames))) { break }295 }296 return (meter.integratedLUFS(), meter.peak)297 }298299 /// Final linear gain: manual gain, plus (when enabled) normalization to -16 LUFS, capped at +20 dB and at a -1 dBFS peak.300 public static func resolveGain(edits: EditDecisionList, measured: (lufs: Double?, peak: Float)) -> Float {301 var gain = pow(10, Float(edits.gainDB) / 20)302 if edits.normalizeLoudness, let lufs = measured.lufs {303 gain *= Float(pow(10, min(20, targetLUFS - lufs) / 20))304 }305 // Amplification never pushes the peak above -1 dBFS; a recording that needs no boost is left untouched.306 if gain > 1, measured.peak > 0 { gain = max(1, min(gain, 0.891 / measured.peak)) }307 return gain308 }309}310