SPB Git

Repository Public License

SpectrogramRendering.swift 12.1 KB 223 lines swift

1//2//  ORIX Audio3//  File: ORIXAudio/Features/Visuals/SpectrogramRendering.swift4//  Description: Spectrogram images: live rolling view, tiled multi-resolution STFT for long files, legend, and high-resolution export.5//6//  Author: Simon-Pierre Boucher7//  Contact: contact@spboucher.ai8//9import CoreGraphics10import ORIXAudioCore11import SwiftUI1213struct SpectrogramDisplay: Equatable, Sendable {14    var palette: Colormap = .orixSpectrum15    var axis: FrequencyAxis = .logarithmic16    /// dB range mapped onto the palette (amplitude dBFS; a full-scale sine is 0 dBFS).17    var floorDB: Float = -10018    var ceilingDB: Float = -1019    var minFrequency: Double = 3020    var maxFrequency: Double = 20_00021    /// Frequency resolution: fs / fftSize.22    var fftSize: Int = 204823}2425enum SpectrogramImage {26    static func make(rgba: [UInt8], width: Int, height: Int) -> CGImage? {27        guard width > 0, height > 0, rgba.count == width * height * 4, let provider = CGDataProvider(data: Data(rgba) as CFData) else { return nil }28        return CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(),29                       bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipLast.rawValue), provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent)30    }3132    static func render(tile: SpectrogramTile, rows: Int, display: SpectrogramDisplay) -> CGImage? {33        let lo = display.axis == .logarithmic ? max(display.minFrequency, 10) : display.minFrequency34        let px = SpectrogramRasterizer.rgba(tile: tile, rows: rows, axis: display.axis, minFrequency: lo, maxFrequency: display.maxFrequency,35                                            floorDB: display.floorDB, ceilingDB: display.ceilingDB, palette: display.palette)36        return make(rgba: px, width: tile.columnCount, height: rows)37    }38}3940/// Colour bar with the dB values it represents.41struct SpectrogramLegend: View {42    let display: SpectrogramDisplay43    var body: some View {44        HStack(spacing: 6) {45            Text(Formatters.dB(Double(display.floorDB), unit: "")).font(.system(size: 9, design: .monospaced))46            LinearGradient(colors: stride(from: 0, through: 255, by: 15).map { i -> Color in47                let l = display.palette.lookupTable48                return Color(red: Double(l[i * 4]) / 255, green: Double(l[i * 4 + 1]) / 255, blue: Double(l[i * 4 + 2]) / 255)49            }, startPoint: .leading, endPoint: .trailing)50            .frame(height: 8).clipShape(Capsule())51            Text("\(Formatters.dB(Double(display.ceilingDB), unit: "dBFS"))").font(.system(size: 9, design: .monospaced))52        }53        .foregroundStyle(ORIX.Palette.plotText)54        .accessibilityElement(children: .ignore)55        .accessibilityLabel(Text("Colour scale from \(Int(display.floorDB)) to \(Int(display.ceilingDB)) decibels full scale, \(display.palette.displayName) palette"))56    }57}5859/// Frequency labels along the left edge, matching the image's axis mapping.60struct FrequencyAxisLabels: View {61    let display: SpectrogramDisplay62    let sampleRate: Double63    var body: some View {64        Canvas { context, size in65            let top = min(display.maxFrequency, sampleRate / 2)66            let lo = display.axis == .logarithmic ? max(display.minFrequency, 10) : display.minFrequency67            let ticks: [Double] = display.axis == .logarithmic ? [50, 100, 200, 500, 1000, 2000, 5000, 10_000, 20_000] : stride(from: 2000.0, through: 20_000, by: 2000).map { $0 }68            for f in ticks where f < top && f > lo {69                let y = size.height * (1 - CGFloat(display.axis.position(of: f, min: lo, max: top)))70                context.stroke(Path { $0.move(to: CGPoint(x: 0, y: y)); $0.addLine(to: CGPoint(x: 5, y: y)) }, with: .color(.white.opacity(0.7)), lineWidth: 1)71                context.draw(Text(f >= 1000 ? "\(Int(f / 1000))k" : "\(Int(f))").font(.system(size: 9, weight: .medium, design: .monospaced)).foregroundStyle(.white.opacity(0.8)),72                             at: CGPoint(x: 8, y: y), anchor: .leading)73            }74        }75        .allowsHitTesting(false)76        .accessibilityHidden(true)77    }78}7980/// Rolling spectrogram of the audio being captured (newest column on the right).81struct LiveSpectrogramView: View {82    let visuals: LiveVisuals83    let display: SpectrogramDisplay84    var isActive = true85    @Environment(\.accessibilityReduceMotion) private var reduceMotion8687    var body: some View {88        GeometryReader { geo in89            TimelineView(.animation(minimumInterval: reduceMotion ? 0.25 : 1.0 / 30, paused: !isActive)) { _ in90                let image = Self.image(visuals: visuals, display: display, columns: min(LiveVisuals.columnHistory, max(64, Int(geo.size.width))), rows: 192)91                ZStack(alignment: .trailing) {92                    ORIX.Palette.plot93                    if let image {94                        Image(decorative: image, scale: 1).resizable().interpolation(.medium)95                            .frame(width: geo.size.width * CGFloat(image.width) / CGFloat(min(LiveVisuals.columnHistory, max(64, Int(geo.size.width)))), height: geo.size.height)96                    }97                    FrequencyAxisLabels(display: display, sampleRate: visuals.read { $0.sampleRate }).frame(maxWidth: .infinity, alignment: .leading)98                }99            }100        }101        .accessibilityElement(children: .ignore)102        .accessibilityLabel(Text("Live spectrogram"))103    }104105    static func image(visuals: LiveVisuals, display: SpectrogramDisplay, columns: Int, rows: Int) -> CGImage? {106        let (values, binCount, sampleRate) = visuals.read { s -> ([Float], Int, Double) in107            let available = s.columns.count / s.binCount108            let take = min(available, columns)109            return (Array(s.columns.suffix(take * s.binCount)), s.binCount, s.sampleRate)110        }111        let count = values.count / max(1, binCount)112        guard count > 0 else { return nil }113        let tile = SpectrogramTile(sampleRate: sampleRate, configuration: LiveVisuals.spectrogramConfiguration, firstColumn: 0, columnCount: count, binCount: binCount, values: values)114        return SpectrogramImage.render(tile: tile, rows: rows, display: display)115    }116}117118/// Computes spectrogram tiles of an audio file on demand, at the resolution the current zoom needs.119///120/// Level L has an effective hop of `fftSize / 4 · 2^L` samples per column. The STFT itself never hops further than one FFT121/// length, and coarser levels take the maximum over consecutive columns, so no part of the audio is skipped and short events122/// stay visible when zoomed out. Tiles hold 256 columns; a multi-hour file is only ever analysed where it is being looked at.123actor SpectrogramTileStore {124    struct Key: Hashable { var level: Int; var index: Int; var fftSize: Int }125    static let tileColumns = 256126    static let cacheLimit = 28127128    let url: URL129    let sampleRate: Double130    let frameCount: Int131    private var reader: AudioFileReader?132    private var cache: [Key: SpectrogramTile] = [:]133    private var order: [Key] = []134135    init?(url: URL) {136        guard let r = try? AudioFileReader(url: url) else { return nil }137        self.url = url; sampleRate = r.sampleRate; frameCount = r.frameCount138    }139140    nonisolated static func baseHop(fftSize: Int) -> Int { fftSize / 4 }141    nonisolated func secondsPerColumn(level: Int, fftSize: Int) -> Double { Double(Self.baseHop(fftSize: fftSize) << level) / sampleRate }142    /// The finest level whose columns are no wider than `secondsPerPoint`… rounded to the level just below one column per point.143    nonisolated func level(forSecondsPerPoint spp: Double, fftSize: Int) -> Int {144        let ratio = spp * sampleRate / Double(Self.baseHop(fftSize: fftSize))145        return ratio <= 1 ? 0 : min(16, Int(log2(ratio)))146    }147    nonisolated func tileDuration(level: Int, fftSize: Int) -> Double { secondsPerColumn(level: level, fftSize: fftSize) * Double(Self.tileColumns) }148149    func tile(_ key: Key) -> SpectrogramTile? {150        if let hit = cache[key] { return hit }151        if reader == nil { reader = try? AudioFileReader(url: url) }152        guard let reader else { return nil }153        let effectiveHop = Self.baseHop(fftSize: key.fftSize) << key.level154        let hop = min(effectiveHop, key.fftSize)155        let pool = effectiveHop / hop156        let firstSample = key.index * Self.tileColumns * effectiveHop157        guard firstSample < frameCount else { return nil }158        // Centre the first window on the tile's first column time where possible.159        let start = max(0, firstSample - key.fftSize / 2)160        let needed = Self.tileColumns * effectiveHop + key.fftSize161        let end = min(frameCount, start + needed)162        let processor = SpectrogramProcessor(sampleRate: sampleRate, configuration: .init(fftSize: key.fftSize, hopSize: hop))163        let binCount = processor.binCount164        var pooled = [Float](repeating: Decibels.floor, count: Self.tileColumns * binCount)165        var produced = 0            // STFT columns so far166        var position = start167        var scratch: [Float] = []168        while position < end, !Task.isCancelled {169            let n = min(262_144, end - position)170            guard let mono = try? reader.readMono(frames: position..<(position + n)), !mono.isEmpty else { break }171            scratch.removeAll(keepingCapacity: true)172            let count = processor.push(mono, into: &scratch)173            scratch.withUnsafeBufferPointer { s in174                pooled.withUnsafeMutableBufferPointer { out in175                    for c in 0..<count {176                        let target = (produced + c) / pool177                        guard target < Self.tileColumns else { break }178                        let src = s.baseAddress! + c * binCount, dst = out.baseAddress! + target * binCount179                        for b in 0..<binCount where src[b] > dst[b] { dst[b] = src[b] }180                    }181                }182            }183            produced += count184            position += mono.count185        }186        let columns = min(Self.tileColumns, (produced + pool - 1) / pool)187        guard columns > 0 else { return nil }188        let tile = SpectrogramTile(sampleRate: sampleRate, configuration: .init(fftSize: key.fftSize, hopSize: effectiveHop), firstColumn: key.index * Self.tileColumns,189                                   columnCount: columns, binCount: binCount, values: Array(pooled[0..<(columns * binCount)]))190        cache[key] = tile; order.append(key)191        if order.count > Self.cacheLimit { cache[order.removeFirst()] = nil }192        return tile193    }194195    /// dB value under a time / frequency position, read from an already computed tile.196    func value(time: Double, frequency: Double, level: Int, fftSize: Int) -> Float? {197        let spc = secondsPerColumn(level: level, fftSize: fftSize)198        let column = Int(time / spc)199        guard let tile = cache[Key(level: level, index: column / Self.tileColumns, fftSize: fftSize)] else { return nil }200        let local = column % Self.tileColumns201        let bin = Int((frequency * Double(fftSize) / sampleRate).rounded())202        guard local < tile.columnCount, bin >= 0, bin < tile.binCount else { return nil }203        return tile.value(column: local, bin: bin)204    }205206    /// Whole-file spectrogram at a fixed pixel width, for image export. Reads the file once, in chunks.207    func overview(width: Int, fftSize: Int) -> SpectrogramTile? {208        let hopNeeded = max(1, frameCount / max(1, width))209        var level = 0210        while (Self.baseHop(fftSize: fftSize) << (level + 1)) <= hopNeeded { level += 1 }211        let effectiveHop = Self.baseHop(fftSize: fftSize) << level212        let tiles = (frameCount / effectiveHop) / Self.tileColumns + 1213        var values: [Float] = []214        var columns = 0, binCount = fftSize / 2 + 1215        for index in 0..<tiles {216            guard let t = tile(Key(level: level, index: index, fftSize: fftSize)) else { break }217            values += t.values; columns += t.columnCount; binCount = t.binCount218        }219        guard columns > 0 else { return nil }220        return SpectrogramTile(sampleRate: sampleRate, configuration: .init(fftSize: fftSize, hopSize: effectiveHop), firstColumn: 0, columnCount: columns, binCount: binCount, values: values)221    }222}223
Files main
Menu

Theme