SPB Git

spb/zyquo-mlx Public MIT

The local MLX foundry for your Mac — run, fine-tune, quantize, and ship models. Nothing leaves your machine.

macosswiftaimlxfine-tuning
Swift 93.4% Python 3.8% Makefile 2.2% Shell 0.5%

phase7: speech pipeline (mlx-whisper via PyBridge + Playground panel + CLI), cancel-state fix, resume/cancel CLI hooks, ffmpeg PATH; matrix progress: speech/cancel/resume/OOM-gate/catalog green

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
simon-pierre boucher committed 11 days ago (Jul 31, 2026) parent e49e298

Showing 7 changed files with +261 and −16

modified Sources/ZyquoMLX/App/CLIFoundry.swift +39 −8
@@ -19,7 +19,7 @@ extension CLI {
19 19 let args = CommandLine.arguments
20 20 return args.contains("--train") || args.contains("--fuse")
21 21 || args.contains("--quantize") || args.contains("--validate-dataset")
22 − || args.contains("--download")
22 + || args.contains("--download") || args.contains("--transcribe")
23 23 }
24 24
25 25 static func runFoundry() async -> Int32 {
@@ -45,6 +45,18 @@ extension CLI {
45 45 try await download(repo: repo)
46 46 return 0
47 47 }
48 + if let modelPath = value(after: "--transcribe", in: args),
49 + let audioPath = value(after: "--audio", in: args)
50 + {
51 + let model = try await ModelStore.shared.describe(
52 + directory: URL(fileURLWithPath: (modelPath as NSString).expandingTildeInPath))
53 + let result = try await SpeechService.shared.transcribe(
54 + model: model,
55 + audio: URL(fileURLWithPath: (audioPath as NSString).expandingTildeInPath))
56 + print("language: \(result.language ?? "?") segments: \(result.segments) took \(String(format: "%.1f", result.duration))s")
57 + print(result.text)
58 + return 0
59 + }
48 60 return 2
49 61 } catch {
50 62 FileHandle.standardError.write(Data("error: \(error.localizedDescription)\n".utf8))
@@ -95,14 +107,33 @@ extension CLI {
95 107 }
96 108
97 109 let service = TrainingService.shared
98 − let run = try await service.createRun(
99 − name: "cli-\(model.name)-\(method.rawValue)",
100 − baseModel: model, dataset: dataset, method: method, hyperParams: hp)
101 − print("run: \(run.id.uuidString)")
102 − print("method: \(method.displayName) on \(model.name)")
103 − print("verdict: \(MemoryAdvisor.trainingVerdict(for: model, method: method, params: hp).displayName)\n")
110 + let resume: Bool
111 + let run: TrainingRun
112 + if let resumeID = value(after: "--resume-run", in: args) {
113 + guard let id = UUID(uuidString: resumeID) else { throw CLIError.usage("bad run id") }
114 + run = try await RunStore.shared.load(id: id)
115 + resume = true
116 + } else {
117 + run = try await service.createRun(
118 + name: "cli-\(model.name)-\(method.rawValue)",
119 + baseModel: model, dataset: dataset, method: method, hyperParams: hp)
120 + resume = false
121 + }
122 + print("run: \(run.id.uuidString)\(resume ? " (warm resume)" : "")")
123 + print("method: \(run.method.displayName) on \(model.name)")
124 + print("verdict: \(MemoryAdvisor.trainingVerdict(for: model, method: run.method, params: run.hyperParams).displayName)\n")
125 +
126 + // Test hook mirroring the UI's Cancel button (drives the same
127 + // TrainingService.cancel() path).
128 + if let cancelAfter = value(after: "--cancel-after", in: args).flatMap(Double.init) {
129 + Task {
130 + try? await Task.sleep(for: .seconds(cancelAfter))
131 + print("\n[--cancel-after \(cancelAfter)s] cancelling…")
132 + await service.cancel()
133 + }
134 + }
104 135
105 − let events = try await service.start(run: run, baseModel: model, dataset: dataset)
136 + let events = try await service.start(run: run, baseModel: model, dataset: dataset, resume: resume)
106 137 for await event in events {
107 138 switch event {
108 139 case .started(let model, let iterations):
added Sources/ZyquoMLX/Engine/SpeechService.swift +69 −0
@@ -0,0 +1,69 @@
1 +//
2 +// SpeechService.swift
3 +// Zyquo MLX
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Mail: contact@spboucher.ai
7 +//
8 +
9 +import Foundation
10 +
11 +/// Transcription result from a speech model.
12 +struct TranscriptionResult: Sendable {
13 + var text: String
14 + var language: String?
15 + var segments: Int
16 + var duration: TimeInterval
17 +}
18 +
19 +/// Speech-to-text via the Python bridge (mlx-whisper — Python-only per
20 +/// docs/MLX-RESEARCH.md §2). Installs the pinned package on first use.
21 +actor SpeechService {
22 +
23 + static let shared = SpeechService()
24 +
25 + /// Exact pin per docs/MLX-RESEARCH.md version snapshot.
26 + static let whisperPin = "mlx-whisper==0.4.3"
27 +
28 + private var whisperInstalled = false
29 +
30 + func transcribe(model: LocalModel, audio: URL) async throws -> TranscriptionResult {
31 + try await ensureWhisper()
32 +
33 + let start = Date()
34 + let stream = try await PythonRunner.shared.stream(
35 + script: "zyquo_transcribe",
36 + arguments: ["--model", model.directory.path, "--audio", audio.path])
37 +
38 + var text = ""
39 + var language: String?
40 + var segments = 0
41 + for try await event in stream {
42 + switch event.event {
43 + case "done":
44 + text = event.string("text") ?? ""
45 + language = event.string("language")
46 + segments = event.int("segments") ?? 0
47 + case "error":
48 + throw PythonRunnerError.processFailed(
49 + exitCode: 1, stderr: event.string("message") ?? "transcription failed")
50 + default:
51 + break
52 + }
53 + }
54 + return TranscriptionResult(
55 + text: text, language: language, segments: segments,
56 + duration: Date().timeIntervalSince(start))
57 + }
58 +
59 + private func ensureWhisper() async throws {
60 + guard !whisperInstalled else { return }
61 + let env = PythonEnvironment.shared
62 + if await !env.isProvisioned {
63 + try await env.provision()
64 + }
65 + // Cheap idempotent install (uv resolves instantly when satisfied).
66 + try await env.install(pins: [Self.whisperPin])
67 + whisperInstalled = true
68 + }
69 +}
modified Sources/ZyquoMLX/PyBridge/PythonRunner.swift +4 −2
@@ -77,9 +77,11 @@ actor PythonRunner {
77 77 let process = Process()
78 78 process.executableURL = python
79 79 process.arguments = ["-u", scriptURL.path] + arguments
80 − // Isolated, reproducible interpreter environment.
80 + // Isolated, reproducible interpreter environment. Homebrew paths are
81 + // included for external tools some pipelines shell out to (ffmpeg
82 + // for mlx-whisper audio decoding).
81 83 process.environment = [
82 − "PATH": "/usr/bin:/bin",
84 + "PATH": "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",
83 85 "HOME": NSHomeDirectory(),
84 86 "PYTHONUNBUFFERED": "1",
85 87 ]
added Sources/ZyquoMLX/PyBridge/scripts/zyquo_transcribe.py +46 −0
@@ -0,0 +1,46 @@
1 +#
2 +# zyquo_transcribe.py
3 +# Zyquo MLX
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Mail: contact@spboucher.ai
7 +#
8 +# Speech-to-text driver over mlx-whisper (docs/MLX-RESEARCH.md §3.2).
9 +# JSON-lines protocol on stdout.
10 +#
11 +# Usage: zyquo_transcribe.py --model <dir-or-repo> --audio <file>
12 +
13 +import argparse
14 +import json
15 +import sys
16 +
17 +
18 +def emit(obj):
19 + sys.stdout.write(json.dumps(obj) + "\n")
20 + sys.stdout.flush()
21 +
22 +
23 +def main():
24 + parser = argparse.ArgumentParser()
25 + parser.add_argument("--model", required=True)
26 + parser.add_argument("--audio", required=True)
27 + args = parser.parse_args()
28 +
29 + try:
30 + emit({"event": "start", "stage": "transcribing", "audio": args.audio})
31 + import mlx_whisper
32 +
33 + result = mlx_whisper.transcribe(args.audio, path_or_hf_repo=args.model)
34 + emit({
35 + "event": "done",
36 + "text": result.get("text", "").strip(),
37 + "language": result.get("language"),
38 + "segments": len(result.get("segments", [])),
39 + })
40 + except Exception as exc: # noqa: BLE001
41 + emit({"event": "error", "message": str(exc), "type": type(exc).__name__})
42 + sys.exit(1)
43 +
44 +
45 +if __name__ == "__main__":
46 + main()
modified Sources/ZyquoMLX/Training/TrainingService.swift +7 −1
@@ -135,7 +135,13 @@ actor TrainingService {
135 135 }
136 136 continuation.yield(event)
137 137 }
138 − current.state = current.lastError == nil ? .completed : .failed
138 + // A cancelled Task ends the stream without throwing — check
139 + // cancellation before declaring completion.
140 + if Task.isCancelled {
141 + current.state = .cancelled
142 + } else {
143 + current.state = current.lastError == nil ? .completed : .failed
144 + }
139 145 } catch is CancellationError {
140 146 current.state = .cancelled
141 147 } catch {
modified Sources/ZyquoMLX/Views/PlaygroundView.swift +87 −4
@@ -36,7 +36,7 @@ struct PlaygroundView: View {
36 36 HStack(spacing: ZyquoTheme.spacing12) {
37 37 Picker("Model", selection: $session.selectedModelID) {
38 38 Text("Choose a model…").tag(String?.none)
39 − ForEach(model.models.filter { $0.type.isSwiftNative }) { item in
39 + ForEach(model.models.filter { $0.type.isSwiftNative || $0.type == .speech }) { item in
40 40 Text("\(item.name) (\(item.type.displayName))").tag(String?.some(item.id))
41 41 }
42 42 }
@@ -73,24 +73,37 @@ struct PlaygroundView: View {
73 73 .padding(.vertical, ZyquoTheme.spacing8)
74 74 .onChange(of: session.selectedModelID) { _, newID in
75 75 guard let newID, let item = model.models.first(where: { $0.id == newID }) else { return }
76 − Task { await session.load(item) }
76 + if item.type == .speech {
77 + // Out-of-process — nothing to load in the engine.
78 + session.speechModel = item
79 + } else {
80 + Task { await session.load(item) }
81 + }
77 82 }
78 83 }
79 84
80 85 @ViewBuilder
81 86 private var content: some View {
82 − switch session.loadedModel?.type {
87 + switch session.loadedModel?.type ?? selectedType {
83 88 case .embedding:
84 89 EmbeddingInspector(session: session)
85 90 case .llm, .vlm:
86 91 ChatPanel(session: session)
92 + case .speech:
93 + SpeechPanel(session: session)
87 94 default:
88 95 EmptyStateView(
89 96 icon: "cpu",
90 97 title: "Pick a Model to Begin",
91 − message: "Choose an installed model above. LLMs and vision models open a streaming chat; embedding models open the vector inspector.")
98 + message: "Choose an installed model above. LLMs and vision models open a streaming chat; embedding models open the vector inspector; speech models open the transcriber.")
92 99 }
93 100 }
101 +
102 + /// Speech models run out-of-process (no in-engine load) — panel selection
103 + /// falls back to the picker choice.
104 + private var selectedType: ModelType? {
105 + model.models.first { $0.id == session.selectedModelID }?.type
106 + }
94 107 }
95 108
96 109 // MARK: - Session state
@@ -117,6 +130,26 @@ final class PlaygroundSession {
117 130 var embedResults: [(text: String, vector: [Float])] = []
118 131 var similarities: [(a: String, b: String, score: Float)] = []
119 132
133 + // Speech state (out-of-process — no engine load)
134 + var speechModel: LocalModel?
135 + var transcription: TranscriptionResult?
136 + var isTranscribing = false
137 +
138 + func transcribe(audio: URL) {
139 + guard let speechModel else { return }
140 + isTranscribing = true
141 + transcription = nil
142 + Task {
143 + do {
144 + transcription = try await SpeechService.shared.transcribe(
145 + model: speechModel, audio: audio)
146 + } catch {
147 + errorMessage = error.localizedDescription
148 + }
149 + isTranscribing = false
150 + }
151 + }
152 +
120 153 func load(_ model: LocalModel) async {
121 154 isLoading = true
122 155 errorMessage = nil
@@ -313,6 +346,56 @@ private struct MessageBubble: View {
313 346 }
314 347 }
315 348
349 +// MARK: - Speech panel
350 +
351 +private struct SpeechPanel: View {
352 + @Bindable var session: PlaygroundSession
353 + @State private var isPickingAudio = false
354 +
355 + var body: some View {
356 + VStack(spacing: ZyquoTheme.spacing16) {
357 + if session.isTranscribing {
358 + ProgressView("Transcribing…")
359 + } else if let result = session.transcription {
360 + ScrollView {
361 + VStack(alignment: .leading, spacing: ZyquoTheme.spacing8) {
362 + HStack {
363 + if let language = result.language {
364 + StatusPill(text: language, color: ZyquoTheme.slate)
365 + }
366 + Text("\(result.segments) segments · \(String(format: "%.1fs", result.duration))")
367 + .font(ZyquoTheme.captionFont)
368 + .foregroundStyle(ZyquoTheme.textSecondary)
369 + }
370 + Text(result.text)
371 + .font(ZyquoTheme.bodyFont)
372 + .textSelection(.enabled)
373 + }
374 + .padding(ZyquoTheme.spacing16)
375 + .frame(maxWidth: .infinity, alignment: .leading)
376 + .zyquoCard()
377 + .padding(ZyquoTheme.spacing20)
378 + }
379 + } else {
380 + EmptyStateView(
381 + icon: "waveform",
382 + title: "Transcribe Audio",
383 + message: "Pick an audio file — the speech model runs locally through the Python pipeline and returns the transcript here.",
384 + actionLabel: "Choose Audio…",
385 + action: { isPickingAudio = true })
386 + }
387 +
388 + if session.transcription != nil {
389 + Button("Transcribe Another…") { isPickingAudio = true }
390 + .padding(.bottom, ZyquoTheme.spacing16)
391 + }
392 + }
393 + .fileImporter(isPresented: $isPickingAudio, allowedContentTypes: [.audio]) { result in
394 + if case .success(let url) = result { session.transcribe(audio: url) }
395 + }
396 + }
397 +}
398 +
316 399 // MARK: - Embedding inspector
317 400
318 401 private struct EmbeddingInspector: View {
modified docs/PLAN.md +9 −1
@@ -162,5 +162,13 @@ Hub pipeline searched, downloaded, and installed a real repo with resumable
162 162 progress. One heuristic fix surfaced by verification: decoder-style embedding
163 163 repos (Qwen3-Embedding ships a `Qwen3ForCausalLM` config) are detected by
164 164 name — the downloaded embedder then produced correct 1024-dim vectors.
165 −## Phase 7 — Verification (not started)
165 +## Phase 7 — Verification (in progress)
166 +
167 +- [ ] Speech pipeline added (mlx-whisper via PyBridge) — closes the model-type matrix gap
168 +- [ ] Inference matrix: LLM (streaming/cancel/memory), embeddings ×2, VLM + image, speech transcription; tok/s + TTFT recorded
169 +- [ ] Training: LoRA + QLoRA to completion, live metrics, checkpoints, warm resume, fuse, behavior change; OOM config blocked by MemoryAdvisor
170 +- [ ] Convert/quantize: HF→MLX (Python path) + Swift-native quantize; outputs load and run; sizes correct
171 +- [ ] Catalog dry-verify: all Featured repo IDs live-checked against the Hub
172 +- [ ] Python env: bootstrap/adopt/repair verified
173 +- [ ] Green results table in docs/VERIFICATION.md (image-gen: documented deferral — upstream FLUX is script-only, no packaged pipeline)
166 174 ## Phase 8 — Signing & Notarization (not started)
167 175