spb/forge-studio Public

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2//3// Contract tests: config codec against the REAL forge configs, the4// param-count formula against `forge info`, LTTB invariants, EMA bias5// correction, CSV parser against real and mutated lines, state machine.6import XCTest7@testable import ForgeStudio89final class ForgeStudioTests: XCTestCase {10    let forgeRepo = URL(fileURLWithPath: NSHomeDirectory() + "/Desktop/forge")1112    func testConfigRoundTripAgainstRealConfigs() throws {13        let configsDir = forgeRepo.appendingPathComponent("configs")14        let files = (try? FileManager.default.contentsOfDirectory(15            at: configsDir, includingPropertiesForKeys: nil))?16            .filter { $0.pathExtension == "json" } ?? []17        try XCTSkipIf(files.isEmpty, "forge repo introuvable")18        for url in files {19            let cfg = try ForgeConfig.load(from: url)20            // Re-export and re-parse: field names must be byte-compatible.21            let data = try cfg.exportJSON()22            let reparsed = try JSONDecoder().decode(ForgeConfig.self, from: data)23            XCTAssertEqual(cfg, reparsed, url.lastPathComponent)24            XCTAssertTrue(cfg.validationErrors.isEmpty,25                          "\(url.lastPathComponent): \(cfg.validationErrors)")26        }27    }2829    func testParamCountMatchesForgeInfo() throws {30        // Known-good values printed by `forge info` for the shipped configs.31        let expected: [(String, Double)] = [32            ("gpt-50m.json", 52.20), ("gpt-50m-moe.json", 151.75),33            ("gpt-50m-deep.json", 50.60), ("gpt-50m-mistral.json", 48.10),34        ]35        for (file, millions) in expected {36            let url = forgeRepo.appendingPathComponent("configs/\(file)")37            guard let cfg = try? ForgeConfig.load(from: url) else { continue }38            XCTAssertEqual(Double(cfg.model.paramCount) / 1e6, millions,39                           accuracy: 0.01, file)40        }41    }4243    func testLTTBInvariants() {44        let pts = (0..<10_000).map {45            Downsampler.XY(x: Double($0), y: sin(Double($0) / 50) + Double($0) * 0.001)46        }47        let ds = Downsampler.lttb(pts, threshold: 500)48        XCTAssertEqual(ds.count, 500)49        XCTAssertEqual(ds.first, pts.first) // endpoints preserved exactly50        XCTAssertEqual(ds.last, pts.last)51        XCTAssertTrue(zip(ds, ds.dropFirst()).allSatisfy { $0.x < $1.x }) // monotonic52        XCTAssertEqual(Downsampler.lttb(pts, threshold: 20_000).count, pts.count)53    }5455    func testEMABiasCorrection() {56        // A constant series must stay exactly constant under bias-corrected EMA.57        let ema = Smoothing.ema([Double](repeating: 3.5, count: 100), smoothing: 0.9)58        XCTAssertTrue(ema.allSatisfy { abs($0 - 3.5) < 1e-9 })59        XCTAssertEqual(Smoothing.ema([1, 2, 3], smoothing: 0), [1, 2, 3])60    }6162    func testCSVParser() {63        var p = LogParser()64        XCTAssertNil(p.parseCSVLine("step,loss,lr,grad_norm,tokens_per_sec,val_loss,elapsed_s"))65        let pt = p.parseCSVLine("584,2.296893,3.680000e-04,0.558962,13692.0,-1.000000,3861.402")66        XCTAssertEqual(pt?.step, 584)67        XCTAssertEqual(pt!.trainLoss, 2.296893, accuracy: 1e-9)68        XCTAssertNil(pt?.valLoss) // -1 sentinel69        XCTAssertEqual(pt!.elapsedS!, 3861.402, accuracy: 1e-6)70        let withVal = p.parseCSVLine("599,2.19,3.05e-04,0.55,13748.0,2.3299,3960.1")71        XCTAssertEqual(withVal!.valLoss!, 2.3299, accuracy: 1e-9)72        // Legacy 6-column header, garbage, truncation: never crash.73        var legacy = LogParser()74        _ = legacy.parseCSVLine("step,loss,lr,grad_norm,tokens_per_sec,val_loss")75        let old = legacy.parseCSVLine("10,5.2,1e-4,0.9,42000.0,-1.0")76        XCTAssertNil(old?.elapsedS)77        XCTAssertNil(legacy.parseCSVLine("garbage,🤖,,"))78        XCTAssertNil(legacy.parseCSVLine("1,2"))79        XCTAssertNil(legacy.parseCSVLine(""))80    }8182    func testStdoutEvents() {83        XCTAssertEqual(84            LogParser.parseStdout("checkpoint saved: runs/x/ckpt_000200.bin"),85            .checkpointSaved(path: "runs/x/ckpt_000200.bin"))86        if case .banner(let params, let steps, let tps, let backend)? =87            LogParser.parseStdout(88                "training gpt-50m: 52196480 params, 1170 steps, 65536 tokens/step, backend=metal, opt=adamw, sched=cosine")89        {90            XCTAssertEqual(params, 52_196_480)91            XCTAssertEqual(steps, 1170)92            XCTAssertEqual(tps, 65536)93            XCTAssertEqual(backend, "metal")94        } else {95            XCTFail("banner non parsé")96        }97    }9899    func testRunStateMachine() {100        // Terminal states allow nothing; queued can't jump to finished.101        XCTAssertTrue(RunState.transitions[.finished]!.isEmpty)102        XCTAssertTrue(RunState.transitions[.failed]!.isEmpty)103        XCTAssertFalse(RunState.transitions[.queued]!.contains(.finished))104        XCTAssertTrue(RunState.transitions[.running]!.contains(.finishing))105    }106107    // M7 stress: 200k synthetic points through the full ingest+snapshot108    // pipeline. The UI reads only snapshots, so these bounds are what keep109    // hover/zoom hitch-free during a 200M-parameter, 100k+-step run.110    func testStress200kPoints() async {111        let store = MetricsStore()112        let clock = ContinuousClock()113114        // Ingest 200k CSV lines through the real parser.115        var csv = "step,loss,lr,grad_norm,tokens_per_sec,val_loss,elapsed_s\n"116        csv.reserveCapacity(12_000_000)117        for i in 0..<200_000 {118            let val = i % 500 == 499 ? String(format: "%.4f", 3.0 + 2.0 * exp(-Double(i) / 30000)) : "-1.0"119            csv += "\(i),\(String(format: "%.4f", 2.5 + 6.0 * exp(-Double(i) / 20000))),3e-4,0.55,14000.0,\(val),\(Double(i) * 4.7)\n"120        }121        let tmp = FileManager.default.temporaryDirectory122            .appendingPathComponent("stress-\(UUID().uuidString).csv")123        try? csv.write(to: tmp, atomically: true, encoding: .utf8)124        defer { try? FileManager.default.removeItem(at: tmp) }125126        let ingest = await clock.measure { await store.ingestCSV(at: tmp) }127        let count = await store.points.count128        XCTAssertEqual(count, 200_000)129        XCTAssertLessThan(ingest, .seconds(5), "ingest 200k: \(ingest)")130131        // Snapshot (LTTB to chart width + EMA) must stay far under a frame132        // budget's worth of off-main-thread work at 10 Hz refresh.133        let snap = await clock.measure {134            _ = await store.snapshot(maxPoints: 1200, smoothing: 0.6)135        }136        XCTAssertLessThan(snap, .milliseconds(250), "snapshot 200k: \(snap)")137        let s = await store.snapshot(maxPoints: 1200, smoothing: 0.6)138        XCTAssertEqual(s.train.count, 1200)139        XCTAssertEqual(s.count, 200_000)140    }141142    func testLRSchedulePreviewMatchesForgeMath() {143        var t = TrainConfig()144        t.lr = 5e-4; t.warmupSteps = 100; t.maxSteps = 1000145        XCTAssertEqual(t.lrAt(step: 0), 5e-4 * 1.0 / 101.0, accuracy: 1e-12)146        XCTAssertEqual(t.lrAt(step: 999) > t.lr * t.minLrRatio, true)147        XCTAssertEqual(t.lrAt(step: 1000), t.lr * t.minLrRatio, accuracy: 1e-12)148        t.schedule = "wsd"149        XCTAssertEqual(t.lrAt(step: 500), t.lr, accuracy: 1e-15) // plateau150    }151}152