Repository Public License
1//2// ORIX Audio3// File: ORIXAudio/Features/Visuals/SpectrogramRendering.swift4// Description: Spectrogram images: live rolling view, tiled multi-resolution STFT for long files, legend, and high-resolution export.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8//9import CoreGraphics10import ORIXAudioCore11import SwiftUI1213struct SpectrogramDisplay: Equatable, Sendable {14 var palette: Colormap = .orixSpectrum15 var axis: FrequencyAxis = .logarithmic16 /// dB range mapped onto the palette (amplitude dBFS; a full-scale sine is 0 dBFS).17 var floorDB: Float = -10018 var ceilingDB: Float = -1019 var minFrequency: Double = 3020 var maxFrequency: Double = 20_00021 /// Frequency resolution: fs / fftSize.22 var fftSize: Int = 204823}2425enum SpectrogramImage {26 static func make(rgba: [UInt8], width: Int, height: Int) -> CGImage? {27 guard width > 0, height > 0, rgba.count == width * height * 4, let provider = CGDataProvider(data: Data(rgba) as CFData) else { return nil }28 return CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(),29 bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipLast.rawValue), provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent)30 }3132 static func render(tile: SpectrogramTile, rows: Int, display: SpectrogramDisplay) -> CGImage? {33 let lo = display.axis == .logarithmic ? max(display.minFrequency, 10) : display.minFrequency34 let px = SpectrogramRasterizer.rgba(tile: tile, rows: rows, axis: display.axis, minFrequency: lo, maxFrequency: display.maxFrequency,35 floorDB: display.floorDB, ceilingDB: display.ceilingDB, palette: display.palette)36 return make(rgba: px, width: tile.columnCount, height: rows)37 }38}3940/// Colour bar with the dB values it represents.41struct SpectrogramLegend: View {42 let display: SpectrogramDisplay43 var body: some View {44 HStack(spacing: 6) {45 Text(Formatters.dB(Double(display.floorDB), unit: "")).font(.system(size: 9, design: .monospaced))46 LinearGradient(colors: stride(from: 0, through: 255, by: 15).map { i -> Color in47 let l = display.palette.lookupTable48 return Color(red: Double(l[i * 4]) / 255, green: Double(l[i * 4 + 1]) / 255, blue: Double(l[i * 4 + 2]) / 255)49 }, startPoint: .leading, endPoint: .trailing)50 .frame(height: 8).clipShape(Capsule())51 Text("\(Formatters.dB(Double(display.ceilingDB), unit: "dBFS"))").font(.system(size: 9, design: .monospaced))52 }53 .foregroundStyle(ORIX.Palette.plotText)54 .accessibilityElement(children: .ignore)55 .accessibilityLabel(Text("Colour scale from \(Int(display.floorDB)) to \(Int(display.ceilingDB)) decibels full scale, \(display.palette.displayName) palette"))56 }57}5859/// Frequency labels along the left edge, matching the image's axis mapping.60struct FrequencyAxisLabels: View {61 let display: SpectrogramDisplay62 let sampleRate: Double63 var body: some View {64 Canvas { context, size in65 let top = min(display.maxFrequency, sampleRate / 2)66 let lo = display.axis == .logarithmic ? max(display.minFrequency, 10) : display.minFrequency67 let ticks: [Double] = display.axis == .logarithmic ? [50, 100, 200, 500, 1000, 2000, 5000, 10_000, 20_000] : stride(from: 2000.0, through: 20_000, by: 2000).map { $0 }68 for f in ticks where f < top && f > lo {69 let y = size.height * (1 - CGFloat(display.axis.position(of: f, min: lo, max: top)))70 context.stroke(Path { $0.move(to: CGPoint(x: 0, y: y)); $0.addLine(to: CGPoint(x: 5, y: y)) }, with: .color(.white.opacity(0.7)), lineWidth: 1)71 context.draw(Text(f >= 1000 ? "\(Int(f / 1000))k" : "\(Int(f))").font(.system(size: 9, weight: .medium, design: .monospaced)).foregroundStyle(.white.opacity(0.8)),72 at: CGPoint(x: 8, y: y), anchor: .leading)73 }74 }75 .allowsHitTesting(false)76 .accessibilityHidden(true)77 }78}7980/// Rolling spectrogram of the audio being captured (newest column on the right).81struct LiveSpectrogramView: View {82 let visuals: LiveVisuals83 let display: SpectrogramDisplay84 var isActive = true85 @Environment(\.accessibilityReduceMotion) private var reduceMotion8687 var body: some View {88 GeometryReader { geo in89 TimelineView(.animation(minimumInterval: reduceMotion ? 0.25 : 1.0 / 30, paused: !isActive)) { _ in90 let image = Self.image(visuals: visuals, display: display, columns: min(LiveVisuals.columnHistory, max(64, Int(geo.size.width))), rows: 192)91 ZStack(alignment: .trailing) {92 ORIX.Palette.plot93 if let image {94 Image(decorative: image, scale: 1).resizable().interpolation(.medium)95 .frame(width: geo.size.width * CGFloat(image.width) / CGFloat(min(LiveVisuals.columnHistory, max(64, Int(geo.size.width)))), height: geo.size.height)96 }97 FrequencyAxisLabels(display: display, sampleRate: visuals.read { $0.sampleRate }).frame(maxWidth: .infinity, alignment: .leading)98 }99 }100 }101 .accessibilityElement(children: .ignore)102 .accessibilityLabel(Text("Live spectrogram"))103 }104105 static func image(visuals: LiveVisuals, display: SpectrogramDisplay, columns: Int, rows: Int) -> CGImage? {106 let (values, binCount, sampleRate) = visuals.read { s -> ([Float], Int, Double) in107 let available = s.columns.count / s.binCount108 let take = min(available, columns)109 return (Array(s.columns.suffix(take * s.binCount)), s.binCount, s.sampleRate)110 }111 let count = values.count / max(1, binCount)112 guard count > 0 else { return nil }113 let tile = SpectrogramTile(sampleRate: sampleRate, configuration: LiveVisuals.spectrogramConfiguration, firstColumn: 0, columnCount: count, binCount: binCount, values: values)114 return SpectrogramImage.render(tile: tile, rows: rows, display: display)115 }116}117118/// Computes spectrogram tiles of an audio file on demand, at the resolution the current zoom needs.119///120/// Level L has an effective hop of `fftSize / 4 · 2^L` samples per column. The STFT itself never hops further than one FFT121/// length, and coarser levels take the maximum over consecutive columns, so no part of the audio is skipped and short events122/// stay visible when zoomed out. Tiles hold 256 columns; a multi-hour file is only ever analysed where it is being looked at.123actor SpectrogramTileStore {124 struct Key: Hashable { var level: Int; var index: Int; var fftSize: Int }125 static let tileColumns = 256126 static let cacheLimit = 28127128 let url: URL129 let sampleRate: Double130 let frameCount: Int131 private var reader: AudioFileReader?132 private var cache: [Key: SpectrogramTile] = [:]133 private var order: [Key] = []134135 init?(url: URL) {136 guard let r = try? AudioFileReader(url: url) else { return nil }137 self.url = url; sampleRate = r.sampleRate; frameCount = r.frameCount138 }139140 nonisolated static func baseHop(fftSize: Int) -> Int { fftSize / 4 }141 nonisolated func secondsPerColumn(level: Int, fftSize: Int) -> Double { Double(Self.baseHop(fftSize: fftSize) << level) / sampleRate }142 /// The finest level whose columns are no wider than `secondsPerPoint`… rounded to the level just below one column per point.143 nonisolated func level(forSecondsPerPoint spp: Double, fftSize: Int) -> Int {144 let ratio = spp * sampleRate / Double(Self.baseHop(fftSize: fftSize))145 return ratio <= 1 ? 0 : min(16, Int(log2(ratio)))146 }147 nonisolated func tileDuration(level: Int, fftSize: Int) -> Double { secondsPerColumn(level: level, fftSize: fftSize) * Double(Self.tileColumns) }148149 func tile(_ key: Key) -> SpectrogramTile? {150 if let hit = cache[key] { return hit }151 if reader == nil { reader = try? AudioFileReader(url: url) }152 guard let reader else { return nil }153 let effectiveHop = Self.baseHop(fftSize: key.fftSize) << key.level154 let hop = min(effectiveHop, key.fftSize)155 let pool = effectiveHop / hop156 let firstSample = key.index * Self.tileColumns * effectiveHop157 guard firstSample < frameCount else { return nil }158 // Centre the first window on the tile's first column time where possible.159 let start = max(0, firstSample - key.fftSize / 2)160 let needed = Self.tileColumns * effectiveHop + key.fftSize161 let end = min(frameCount, start + needed)162 let processor = SpectrogramProcessor(sampleRate: sampleRate, configuration: .init(fftSize: key.fftSize, hopSize: hop))163 let binCount = processor.binCount164 var pooled = [Float](repeating: Decibels.floor, count: Self.tileColumns * binCount)165 var produced = 0 // STFT columns so far166 var position = start167 var scratch: [Float] = []168 while position < end, !Task.isCancelled {169 let n = min(262_144, end - position)170 guard let mono = try? reader.readMono(frames: position..<(position + n)), !mono.isEmpty else { break }171 scratch.removeAll(keepingCapacity: true)172 let count = processor.push(mono, into: &scratch)173 scratch.withUnsafeBufferPointer { s in174 pooled.withUnsafeMutableBufferPointer { out in175 for c in 0..<count {176 let target = (produced + c) / pool177 guard target < Self.tileColumns else { break }178 let src = s.baseAddress! + c * binCount, dst = out.baseAddress! + target * binCount179 for b in 0..<binCount where src[b] > dst[b] { dst[b] = src[b] }180 }181 }182 }183 produced += count184 position += mono.count185 }186 let columns = min(Self.tileColumns, (produced + pool - 1) / pool)187 guard columns > 0 else { return nil }188 let tile = SpectrogramTile(sampleRate: sampleRate, configuration: .init(fftSize: key.fftSize, hopSize: effectiveHop), firstColumn: key.index * Self.tileColumns,189 columnCount: columns, binCount: binCount, values: Array(pooled[0..<(columns * binCount)]))190 cache[key] = tile; order.append(key)191 if order.count > Self.cacheLimit { cache[order.removeFirst()] = nil }192 return tile193 }194195 /// dB value under a time / frequency position, read from an already computed tile.196 func value(time: Double, frequency: Double, level: Int, fftSize: Int) -> Float? {197 let spc = secondsPerColumn(level: level, fftSize: fftSize)198 let column = Int(time / spc)199 guard let tile = cache[Key(level: level, index: column / Self.tileColumns, fftSize: fftSize)] else { return nil }200 let local = column % Self.tileColumns201 let bin = Int((frequency * Double(fftSize) / sampleRate).rounded())202 guard local < tile.columnCount, bin >= 0, bin < tile.binCount else { return nil }203 return tile.value(column: local, bin: bin)204 }205206 /// Whole-file spectrogram at a fixed pixel width, for image export. Reads the file once, in chunks.207 func overview(width: Int, fftSize: Int) -> SpectrogramTile? {208 let hopNeeded = max(1, frameCount / max(1, width))209 var level = 0210 while (Self.baseHop(fftSize: fftSize) << (level + 1)) <= hopNeeded { level += 1 }211 let effectiveHop = Self.baseHop(fftSize: fftSize) << level212 let tiles = (frameCount / effectiveHop) / Self.tileColumns + 1213 var values: [Float] = []214 var columns = 0, binCount = fftSize / 2 + 1215 for index in 0..<tiles {216 guard let t = tile(Key(level: level, index: index, fftSize: fftSize)) else { break }217 values += t.values; columns += t.columnCount; binCount = t.binCount218 }219 guard columns > 0 else { return nil }220 return SpectrogramTile(sampleRate: sampleRate, configuration: .init(fftSize: fftSize, hopSize: effectiveHop), firstColumn: 0, columnCount: columns, binCount: binCount, values: values)221 }222}223