spb/forge-studio Public

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2//3// Datasets tab: the registered datasets (token counts, vocab, size) and the4// "new dataset" flow driving tools/prepare_data.py (TinyStories) or5// tools/prepare_hf_data.py (HF sources & research presets) with a live6// progress console. Python and the tools live in the forge repo, located7// from the configured binary.8import SwiftUI910struct DatasetsView: View {11    @Environment(AppModel.self) private var app12    @State private var datasets: [Dataset] = []13    @State private var showNew = false1415    var body: some View {16        VStack(alignment: .leading, spacing: 12) {17            HStack {18                Text("Datasets").font(.title2.weight(.semibold))19                Spacer()20                Button {21                    showNew = true22                } label: {23                    Label("Nouveau dataset", systemImage: "plus")24                }25            }26            if datasets.isEmpty {27                ContentUnavailableView(28                    "Aucun dataset",29                    systemImage: "cylinder.split.1x2",30                    description: Text("Préparez TinyStories ou un mélange Hugging Face — tout se passe ici, sans terminal."))31            } else {32                Table(datasets) {33                    TableColumn("Nom") { Text($0.name) }34                    TableColumn("Tokens train") {35                        Text($0.trainTokens?.formatted(.number.notation(.compactName)) ?? "?")36                    }37                    TableColumn("Tokens val") {38                        Text($0.valTokens?.formatted(.number.notation(.compactName)) ?? "?")39                    }40                    TableColumn("Vocab") { Text($0.vocabSize.map(String.init) ?? "?") }41                    TableColumn("Taille") {42                        Text(ByteCountFormatter.string(fromByteCount: $0.sizeOnDisk,43                                                       countStyle: .file))44                    }45                    TableColumn("Chemin") {46                        Text($0.path).font(.caption.monospaced())47                            .foregroundStyle(.secondary)48                    }49                }50            }51        }52        .padding()53        .onAppear { datasets = app.datasets() }54        .sheet(isPresented: $showNew, onDismiss: { datasets = app.datasets() }) {55            NewDatasetSheet()56        }57        .navigationTitle("Datasets")58    }59}6061struct NewDatasetSheet: View {62    @Environment(AppModel.self) private var app63    @Environment(\.dismiss) private var dismiss6465    enum Kind: String, CaseIterable {66        case tinystories = "TinyStories"67        case hf = "Hugging Face"68    }6970    @State private var kind: Kind = .tinystories71    @State private var name = "tinystories"72    @State private var vocabSize = 409673    @State private var maxTrainMB = 100.074    @State private var hfChoice = "preset:smollm-web"75    @State private var console: [String] = []76    @State private var running = false77    @State private var failed = false7879    // Mirrors SOURCES/PRESETS in tools/prepare_hf_data.py.80    private let hfChoices: [(label: String, value: String)] = [81        ("Preset · smollm-web (60% FineWeb-Edu + 40% DCLM)", "preset:smollm-web"),82        ("Preset · textbooks (FineWeb-Edu + Cosmopedia)", "preset:textbooks"),83        ("Preset · smol-full (edu+cosmo+math)", "preset:smol-full"),84        ("Preset · decay-anneal (fin de run WSD)", "preset:decay-anneal"),85        ("Source · fineweb-edu", "source:fineweb-edu"),86        ("Source · cosmopedia", "source:cosmopedia"),87        ("Source · finemath", "source:finemath"),88        ("Source · wikipedia-fr", "source:wikipedia-fr"),89        ("Source · tinystories (via HF)", "source:tinystories"),90    ]9192    var body: some View {93        VStack(alignment: .leading, spacing: 12) {94            Text("Nouveau dataset").font(.title2.weight(.semibold))95            Picker("Type", selection: $kind) {96                ForEach(Kind.allCases, id: \.self) { Text($0.rawValue) }97            }98            .pickerStyle(.segmented)99            Form {100                TextField("Nom (dossier dans data/)", text: $name)101                if kind == .hf {102                    Picker("Corpus", selection: $hfChoice) {103                        ForEach(hfChoices, id: \.value) { Text($0.label).tag($0.value) }104                    }105                }106                TextField("vocab_size", value: $vocabSize, format: .number)107                TextField("Taille max train (MB)", value: $maxTrainMB, format: .number)108            }109            .formStyle(.grouped)110            .frame(height: kind == .hf ? 180 : 150)111            .disabled(running)112113            if !console.isEmpty {114                ConsoleView(lines: console)115            }116117            HStack {118                if failed {119                    Label("Échec — voir la console.", systemImage: "xmark.circle")120                        .foregroundStyle(.red)121                }122                Spacer()123                Button(running ? "Fermer (continue en arrière-plan)" : "Fermer") {124                    dismiss()125                }126                Button(running ? "Préparation…" : "Préparer") { prepare() }127                    .keyboardShortcut(.defaultAction)128                    .disabled(running || name.isEmpty)129            }130        }131        .padding()132        .frame(minWidth: 640, minHeight: 480)133    }134135    private var forgeRepo: URL? {136        // <repo>/build/forge → <repo>137        ForgeBinaryLocator.savedBinaryURL?138            .deletingLastPathComponent().deletingLastPathComponent()139    }140141    private func prepare() {142        guard let repo = forgeRepo else {143            console = ["⚠︎ Binaire forge non configuré — Réglages."]144            failed = true145            return146        }147        let outDir = app.store.workspaceURL.appendingPathComponent("data/\(name)")148        var args: [String]149        switch kind {150        case .tinystories:151            args = ["tools/prepare_data.py", "--out", outDir.path,152                    "--vocab-size", String(vocabSize),153                    "--max-train-mb", String(maxTrainMB)]154        case .hf:155            let parts = hfChoice.split(separator: ":", maxSplits: 1).map(String.init)156            args = ["tools/prepare_hf_data.py", "--\(parts[0])", parts[1],157                    "--out", outDir.path, "--vocab-size", String(vocabSize),158                    "--max-train-mb", String(maxTrainMB)]159        }160        running = true161        failed = false162        console = ["$ python3 " + args.joined(separator: " ")]163        Task {164            do {165                let runner = ProcessRunner()166                let (lines, exit) = try await runner.launch(167                    executable: URL(fileURLWithPath: "/usr/bin/env"),168                    arguments: ["python3"] + args,169                    currentDirectory: repo)170                for await (line, isErr) in lines {171                    console.append(isErr ? "⚠︎ " + line : line)172                    if console.count > 2000 { console.removeFirst(500) }173                }174                let status = await exit.value175                running = false176                failed = status.code != 0177                if !failed { console.append("✓ Dataset prêt : \(outDir.path)") }178            } catch {179                running = false180                failed = true181                console.append("⚠︎ " + error.localizedDescription)182            }183        }184    }185}186