spb/forge-studio Public

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2//3// Multi-run overlay: val-loss (and optional train-EMA) curves for 2–8 runs,4// X axis in steps, TOKENS (the honest axis when batch sizes differ) or5// wall-clock; legend annotated with the auto-computed hyperparameter diff6// (only the fields that differ across the selection); summary table + CSV7// export.8import Charts9import SwiftUI1011struct CompareView: View {12    @Environment(AppModel.self) private var app13    @State private var selected: Set<UUID> = []14    @State private var xMode: XMode = .steps15    @State private var showTrainEMA = false16    @State private var series: [UUID: CompareSeries] = [:]1718    enum XMode: String, CaseIterable {19        case steps = "steps"20        case tokens = "tokens"21        case time = "temps"22    }2324    struct CompareSeries {25        var val: [(x: Double, y: Double)] = []26        var ema: [(x: Double, y: Double)] = []27        var bestVal: Double?28        var finalPpl: Double?29        var meanTps: Double?30        var totalTime: Double?31    }3233    private let palette: [Color] = [.blue, .orange, .green, .purple, .red, .teal,34                                    .pink, .brown]3536    private var candidates: [Run] {37        app.store.runs.filter { FileManager.default.fileExists(atPath: $0.logCSVPath) }38            .sorted { $0.createdAt > $1.createdAt }39    }40    private var selectedRuns: [Run] { candidates.filter { selected.contains($0.id) } }4142    var body: some View {43        HSplitView {44            List(candidates, selection: $selected) { run in45                RunRow(run: run).tag(run.id)46            }47            .frame(minWidth: 230, maxWidth: 300)4849            VStack(alignment: .leading, spacing: 12) {50                HStack {51                    Text("Comparaison").font(.title2.weight(.semibold))52                    Spacer()53                    Toggle("train EMA", isOn: $showTrainEMA).toggleStyle(.checkbox)54                    Picker("", selection: $xMode) {55                        ForEach(XMode.allCases, id: \.self) { Text($0.rawValue) }56                    }57                    .pickerStyle(.segmented)58                    .frame(width: 220)59                    Button("Exporter CSV") { exportCSV() }60                        .disabled(selectedRuns.isEmpty)61                }62                if selectedRuns.count < 2 {63                    ContentUnavailableView(64                        "Sélectionnez 2 à 8 runs",65                        systemImage: "chart.xyaxis.line",66                        description: Text("⌘-clic dans la liste pour comparer leurs courbes de validation."))67                } else {68                    diffLegend69                    chart70                    summaryTable71                }72                Spacer(minLength: 0)73            }74            .padding()75        }76        .task(id: "\(selected.hashValue)|\(xMode.rawValue)") { await loadSeries() }77        .navigationTitle("Comparer")78    }7980    private var chart: some View {81        Chart {82            ForEach(Array(selectedRuns.enumerated()), id: \.element.id) { idx, run in83                let s = series[run.id]84                ForEach(s?.val ?? [], id: \.x) { p in85                    LineMark(x: .value("x", p.x), y: .value("val", p.y),86                             series: .value("run", run.name + " val"))87                        .foregroundStyle(palette[idx % palette.count])88                }89                if showTrainEMA {90                    ForEach(s?.ema ?? [], id: \.x) { p in91                        LineMark(x: .value("x", p.x), y: .value("ema", p.y),92                                 series: .value("run", run.name + " ema"))93                            .foregroundStyle(palette[idx % palette.count].opacity(0.4))94                    }95                }96            }97        }98        .chartXAxisLabel(xMode == .tokens ? "tokens vus"99                         : xMode == .time ? "secondes" : "steps")100        .frame(minHeight: 300)101    }102103    // Only the config fields that differ across the selection.104    private var diffLegend: some View {105        let diffs = configDiffs()106        return VStack(alignment: .leading, spacing: 4) {107            ForEach(Array(selectedRuns.enumerated()), id: \.element.id) { idx, run in108                HStack(spacing: 8) {109                    Circle().fill(palette[idx % palette.count]).frame(width: 9, height: 9)110                    Text(run.name).font(.callout.weight(.medium))111                    Text(diffs[run.id] ?? "").font(.caption)112                        .foregroundStyle(.secondary)113                }114            }115        }116    }117118    private func configDiffs() -> [UUID: String] {119        guard selectedRuns.count >= 2 else { return [:] }120        // Flatten each config to key: value strings via JSON.121        var flat: [UUID: [String: String]] = [:]122        for run in selectedRuns {123            guard let data = try? run.config.exportJSON(),124                  let obj = try? JSONSerialization.jsonObject(with: data)125                      as? [String: [String: Any]] else { continue }126            var kv: [String: String] = [:]127            for (section, fields) in obj {128                for (k, v) in fields { kv["\(section).\(k)"] = "\(v)" }129            }130            flat[run.id] = kv131        }132        let allKeys = Set(flat.values.flatMap(\.keys))133        let differing = allKeys.filter { key in134            Set(flat.values.map { $0[key] ?? "" }).count > 1135        }.sorted()136        var out: [UUID: String] = [:]137        for run in selectedRuns {138            out[run.id] = differing139                .filter { $0 != "model.name" }140                .map { "\($0.split(separator: ".").last!)=\(flat[run.id]?[$0] ?? "?")" }141                .joined(separator: "  ")142        }143        return out144    }145146    private var summaryTable: some View {147        Table(selectedRuns) {148            TableColumn("Run") { Text($0.name) }149            TableColumn("Params") {150                Text($0.config.model.paramCount.formatted(.number.notation(.compactName)))151            }152            TableColumn("Best val") { run in153                Text(series[run.id]?.bestVal.map { String(format: "%.4f", $0) } ?? "—")154            }155            TableColumn("PPL finale") { run in156                Text(series[run.id]?.finalPpl.map { String(format: "%.1f", $0) } ?? "—")157            }158            TableColumn("tok/s moyen") { run in159                Text(series[run.id]?.meanTps.map { String(format: "%.0f", $0) } ?? "—")160            }161            TableColumn("Durée") { run in162                Text(series[run.id]?.totalTime.map {163                    Duration.seconds($0).formatted(.time(pattern: .hourMinuteSecond))164                } ?? "—")165            }166        }167        .frame(height: CGFloat(40 + selectedRuns.count * 28))168    }169170    private func loadSeries() async {171        var out: [UUID: CompareSeries] = [:]172        for run in selectedRuns {173            let store = MetricsStore()174            await store.ingestCSV(at: URL(fileURLWithPath: run.logCSVPath))175            let snap = await store.snapshot(maxPoints: 800, smoothing: 0.6)176            let tps = Double(run.config.tokensPerStep)177            func xform(_ p: Downsampler.XY, elapsedLookup: Bool = false) -> (Double, Double) {178                switch xMode {179                case .steps: return (p.x, p.y)180                case .tokens: return ((p.x + 1) * tps, p.y)181                case .time: return (p.x, p.y) // refined below with elapsed182                }183            }184            var s = CompareSeries()185            if xMode == .time {186                // elapsed_s per step from the raw points.187                let pts = await store.points188                s.val = pts.compactMap { p in189                    guard let v = p.valLoss, let e = p.elapsedS else { return nil }190                    return (e, v)191                }192                s.ema = zip(pts, Smoothing.ema(pts.map(\.trainLoss), smoothing: 0.6))193                    .compactMap { p, e in p.elapsedS.map { ($0, e) } }194            } else {195                s.val = snap.val.map { xform($0) }196                s.ema = snap.trainEMA.map { xform($0) }197            }198            s.bestVal = snap.bestVal?.loss199            s.finalPpl = snap.lastPoint.map { exp($0.trainLoss) }200            let allPts = await store.points201            if !allPts.isEmpty {202                s.meanTps = allPts.map(\.tokensPerSec).reduce(0, +) / Double(allPts.count)203                s.totalTime = allPts.last?.elapsedS204            }205            out[run.id] = s206        }207        series = out208    }209210    private func exportCSV() {211        var csv = "run,params,best_val,final_ppl,mean_tok_s,total_s\n"212        for run in selectedRuns {213            let s = series[run.id]214            csv += "\(run.name),\(run.config.model.paramCount),"215            csv += "\(s?.bestVal.map { String($0) } ?? ""),"216            csv += "\(s?.finalPpl.map { String($0) } ?? ""),"217            csv += "\(s?.meanTps.map { String($0) } ?? ""),"218            csv += "\(s?.totalTime.map { String($0) } ?? "")\n"219        }220        let panel = NSSavePanel()221        panel.nameFieldStringValue = "compare.csv"222        if panel.runModal() == .OK, let url = panel.url {223            try? csv.write(to: url, atomically: true, encoding: .utf8)224        }225    }226}227