spb/forge-studio

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2//3// Contract tests: config codec against the REAL forge configs, the4// param-count formula against `forge info`, LTTB invariants, EMA bias5// correction, CSV parser against real and mutated lines, state machine.6import XCTest7@testable import ForgeStudio89final class ForgeStudioTests: XCTestCase {10    let forgeRepo = URL(fileURLWithPath: NSHomeDirectory() + "/Desktop/forge")1112    func testConfigRoundTripAgainstRealConfigs() throws {13        let configsDir = forgeRepo.appendingPathComponent("configs")14        let files = (try? FileManager.default.contentsOfDirectory(15            at: configsDir, includingPropertiesForKeys: nil))?16            .filter { $0.pathExtension == "json" } ?? []17        try XCTSkipIf(files.isEmpty, "forge repo introuvable")18        for url in files {19            let cfg = try ForgeConfig.load(from: url)20            // Re-export and re-parse: field names must be byte-compatible.21            let data = try cfg.exportJSON()22            let reparsed = try JSONDecoder().decode(ForgeConfig.self, from: data)23            XCTAssertEqual(cfg, reparsed, url.lastPathComponent)24            XCTAssertTrue(cfg.validationErrors.isEmpty,25                          "\(url.lastPathComponent): \(cfg.validationErrors)")26        }27    }2829    func testParamCountMatchesForgeInfo() throws {30        // Known-good values printed by `forge info` for the shipped configs.31        let expected: [(String, Double)] = [32            ("gpt-50m.json", 52.20), ("gpt-50m-moe.json", 151.75),33            ("gpt-50m-deep.json", 50.60), ("gpt-50m-mistral.json", 48.10),34        ]35        for (file, millions) in expected {36            let url = forgeRepo.appendingPathComponent("configs/\(file)")37            guard let cfg = try? ForgeConfig.load(from: url) else { continue }38            XCTAssertEqual(Double(cfg.model.paramCount) / 1e6, millions,39                           accuracy: 0.01, file)40        }41    }4243    func testLTTBInvariants() {44        let pts = (0..<10_000).map {45            Downsampler.XY(x: Double($0), y: sin(Double($0) / 50) + Double($0) * 0.001)46        }47        let ds = Downsampler.lttb(pts, threshold: 500)48        XCTAssertEqual(ds.count, 500)49        XCTAssertEqual(ds.first, pts.first) // endpoints preserved exactly50        XCTAssertEqual(ds.last, pts.last)51        XCTAssertTrue(zip(ds, ds.dropFirst()).allSatisfy { $0.x < $1.x }) // monotonic52        XCTAssertEqual(Downsampler.lttb(pts, threshold: 20_000).count, pts.count)53    }5455    func testEMABiasCorrection() {56        // A constant series must stay exactly constant under bias-corrected EMA.57        let ema = Smoothing.ema([Double](repeating: 3.5, count: 100), smoothing: 0.9)58        XCTAssertTrue(ema.allSatisfy { abs($0 - 3.5) < 1e-9 })59        XCTAssertEqual(Smoothing.ema([1, 2, 3], smoothing: 0), [1, 2, 3])60    }6162    func testCSVParser() {63        var p = LogParser()64        XCTAssertNil(p.parseCSVLine("step,loss,lr,grad_norm,tokens_per_sec,val_loss,elapsed_s"))65        let pt = p.parseCSVLine("584,2.296893,3.680000e-04,0.558962,13692.0,-1.000000,3861.402")66        XCTAssertEqual(pt?.step, 584)67        XCTAssertEqual(pt!.trainLoss, 2.296893, accuracy: 1e-9)68        XCTAssertNil(pt?.valLoss) // -1 sentinel69        XCTAssertEqual(pt!.elapsedS!, 3861.402, accuracy: 1e-6)70        let withVal = p.parseCSVLine("599,2.19,3.05e-04,0.55,13748.0,2.3299,3960.1")71        XCTAssertEqual(withVal!.valLoss!, 2.3299, accuracy: 1e-9)72        // Legacy 6-column header, garbage, truncation: never crash.73        var legacy = LogParser()74        _ = legacy.parseCSVLine("step,loss,lr,grad_norm,tokens_per_sec,val_loss")75        let old = legacy.parseCSVLine("10,5.2,1e-4,0.9,42000.0,-1.0")76        XCTAssertNil(old?.elapsedS)77        XCTAssertNil(legacy.parseCSVLine("garbage,🤖,,"))78        XCTAssertNil(legacy.parseCSVLine("1,2"))79        XCTAssertNil(legacy.parseCSVLine(""))80    }8182    func testStdoutEvents() {83        XCTAssertEqual(84            LogParser.parseStdout("checkpoint saved: runs/x/ckpt_000200.bin"),85            .checkpointSaved(path: "runs/x/ckpt_000200.bin"))86        if case .banner(let params, let steps, let tps, let backend)? =87            LogParser.parseStdout(88                "training gpt-50m: 52196480 params, 1170 steps, 65536 tokens/step, backend=metal, opt=adamw, sched=cosine")89        {90            XCTAssertEqual(params, 52_196_480)91            XCTAssertEqual(steps, 1170)92            XCTAssertEqual(tps, 65536)93            XCTAssertEqual(backend, "metal")94        } else {95            XCTFail("banner non parsé")96        }97    }9899    func testRunStateMachine() {100        // Terminal states allow nothing; queued can't jump to finished.101        XCTAssertTrue(RunState.transitions[.finished]!.isEmpty)102        XCTAssertTrue(RunState.transitions[.failed]!.isEmpty)103        XCTAssertFalse(RunState.transitions[.queued]!.contains(.finished))104        XCTAssertTrue(RunState.transitions[.running]!.contains(.finishing))105    }106107    func testLRSchedulePreviewMatchesForgeMath() {108        var t = TrainConfig()109        t.lr = 5e-4; t.warmupSteps = 100; t.maxSteps = 1000110        XCTAssertEqual(t.lrAt(step: 0), 5e-4 * 1.0 / 101.0, accuracy: 1e-12)111        XCTAssertEqual(t.lrAt(step: 999) > t.lr * t.minLrRatio, true)112        XCTAssertEqual(t.lrAt(step: 1000), t.lr * t.minLrRatio, accuracy: 1e-12)113        t.schedule = "wsd"114        XCTAssertEqual(t.lrAt(step: 500), t.lr, accuracy: 1e-15) // plateau115    }116}117