Repository Public
1import { execFile } from "node:child_process";2import { existsSync } from "node:fs";3import { mkdtemp, rm, writeFile } from "node:fs/promises";4import os from "node:os";5import path from "node:path";6import type { Lang, WordTiming } from "../lib/contracts";7import { env } from "../lib/env";8import { alignScript, type AlignedWord } from "./align";9import { ff, probe } from "./ffmpeg";1011/**12 * Voix et transcription locales.13 * - TTS : Kokoro v1.0 par `kokoro-onnx` (py/tts.py ; voix `ff_siwis` en français, `af_heart` / `am_michael` en anglais) ; repli `say` (macOS).14 * - Transcription et alignement par mot : Whisper large-v3-turbo par `mlx-whisper` (py/transcribe.py).15 * Les scripts Python sont lancés par `uv run --project <env.pyDir> python …` et répondent en JSON sur stdout.16 * Premier lancement sur un nœud : uv crée l'environnement (≈ 1 Go de paquets) et les modèles sont téléchargés17 * (Kokoro ≈ 340 Mo dans ~/.cache/sp-clip/kokoro, Whisper ≈ 1,6 Go dans le cache Hugging Face) : délais jusqu'à 30 min tolérés.18 */1920const PATHS = `${os.homedir()}/.local/bin:/opt/homebrew/bin:/usr/local/bin:${process.env.PATH ?? ""}`;21const FIRST_RUN_TIMEOUT = 30 * 60_000;2223export const KOKORO_VOICES: Record<Lang, string[]> = { fr: ["ff_siwis"], en: ["af_heart", "am_michael"] };24const SAY_VOICES: Record<Lang, string[]> = { fr: ["Amélie", "Thomas", "Audrey", "Aurélie", "Jacques", "Flo", "Eddy"], en: ["Samantha", "Alex", "Daniel", "Karen", "Moira", "Tessa", "Flo", "Eddy", "Fred"] };2526function exec(bin: string, args: string[], timeoutMs: number): Promise<{ stdout: string; stderr: string; code: number }> {27 return new Promise((resolve, reject) => {28 execFile(bin, args, { env: { ...process.env, PATH: PATHS, PYTHONUNBUFFERED: "1", HF_HUB_DISABLE_PROGRESS_BARS: "1" }, maxBuffer: 256 * 1024 * 1024, timeout: timeoutMs }, (err, stdout, stderr) => {29 if (err && (err as NodeJS.ErrnoException).code === "ENOENT") return reject(new Error(`${bin} introuvable dans le PATH`));30 if (err && err.killed) return reject(new Error(`${bin} : délai dépassé (${Math.round(timeoutMs / 1000)} s)`));31 resolve({ stdout, stderr, code: err ? (typeof err.code === "number" ? err.code : 1) : 0 });32 });33 });34}3536/** Lance un script de py/ et renvoie son JSON. Erreur explicite si uv, l'environnement ou le script échouent. */37async function runPy<T extends { ok: boolean; error?: string }>(script: string, args: string[], timeoutMs = FIRST_RUN_TIMEOUT): Promise<T> {38 const dir = env.pyDir;39 if (!existsSync(path.join(dir, script))) throw new Error(`Script Python manquant : ${path.join(dir, script)} (CLIP_PY_DIR)`);40 const r = await exec("uv", ["run", "--quiet", "--project", dir, "python", path.join(dir, script), ...args], timeoutMs);41 const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();42 let parsed: T | null = null;43 if (line) { try { parsed = JSON.parse(line) as T; } catch { parsed = null; } }44 if (!parsed) throw new Error(`${script} : pas de réponse JSON (code ${r.code}) — ${r.stderr.trim().split("\n").slice(-3).join(" | ") || "aucune sortie"}`);45 return parsed;46}4748/** Voix installées : nom complet tel que listé par `say -v ?` (ex. « Amélie (Français (Canada)) ») et langue (ex. fr_CA). */49async function sayVoices(): Promise<{ name: string; locale: string }[]> {50 if (process.platform !== "darwin" || !existsSync("/usr/bin/say")) return [];51 try {52 const { stdout } = await exec("/usr/bin/say", ["-v", "?"], 30_000);53 const seen = new Set<string>();54 return stdout.split("\n").flatMap((l) => {55 const m = l.match(/^(.+?)\s+([a-z]{2,3}_[A-Za-z0-9]+)\s+#/);56 if (!m || seen.has(m[1].trim())) return [];57 seen.add(m[1].trim());58 return [{ name: m[1].trim(), locale: m[2] }];59 });60 } catch { return []; }61}62/** Choisit une voix `say` : celle demandée si elle existe, sinon la première voix préférée de la langue, sinon n'importe quelle voix de la langue. */63function pickSayVoice(voices: { name: string; locale: string }[], voiceId: string, language: Lang): string | undefined {64 const base = (n: string) => n.replace(/\s*\(.*$/, "");65 const asked = voiceId.trim() && voices.find((v) => v.name === voiceId || base(v.name) === voiceId);66 if (asked) return asked.name;67 const inLang = voices.filter((v) => v.locale.startsWith(`${language}_`));68 for (const pref of SAY_VOICES[language]) { const v = inLang.find((x) => base(x.name) === pref); if (v) return v.name; }69 return inLang[0]?.name;70}7172let capsCache: { at: number; value: Awaited<ReturnType<typeof speechCapabilities>> } | null = null;7374/** Rapporte ce qui est réellement installé sur ce nœud (paquets importables, modèles déjà téléchargés, voix `say`). */75export async function speechCapabilities(): Promise<{ kokoro: boolean; whisper: boolean; say: boolean; detail: Record<string, string> }> {76 if (capsCache && Date.now() - capsCache.at < 60_000) return capsCache.value;77 const detail: Record<string, string> = { pyDir: env.pyDir };78 let kokoro = false, whisper = false;79 try {80 const uv = await exec("uv", ["--version"], 15_000);81 detail.uv = uv.stdout.trim() || "uv présent";82 try {83 const k = await runPy<{ ok: boolean; error?: string; package?: string; models?: boolean; cache?: string }>("tts.py", ["--check"]);84 kokoro = k.ok;85 detail.kokoro = k.ok ? `${k.package} ; modèles ${k.models ? "présents" : "à télécharger au premier usage (≈ 340 Mo)"} dans ${k.cache} ; voix ${[...KOKORO_VOICES.fr, ...KOKORO_VOICES.en].join(", ")}` : `indisponible : ${k.error}`;86 } catch (e) { detail.kokoro = `indisponible : ${(e as Error).message}`; }87 try {88 const w = await runPy<{ ok: boolean; error?: string; package?: string; model?: string; modelCached?: boolean }>("transcribe.py", ["--check"]);89 whisper = w.ok;90 detail.whisper = w.ok ? `${w.package} ; modèle ${w.model} ${w.modelCached ? "en cache" : "à télécharger au premier usage (≈ 1,6 Go)"}` : `indisponible : ${w.error}`;91 } catch (e) { detail.whisper = `indisponible : ${(e as Error).message}`; }92 } catch (e) {93 detail.uv = `indisponible : ${(e as Error).message}`;94 detail.kokoro = detail.whisper = "indisponible : uv requis";95 }96 const voices = await sayVoices();97 detail.say = voices.length ? `${voices.length} voix système (fr : ${pickSayVoice(voices, "", "fr") ?? "aucune"} ; en : ${pickSayVoice(voices, "", "en") ?? "aucune"})` : "indisponible (macOS requis)";98 const value = { kokoro, whisper, say: voices.length > 0, detail };99 capsCache = { at: Date.now(), value };100 return value;101}102103async function toWav48k(input: string, output: string): Promise<number> {104 await ff(["-i", input, "-vn", "-af", "aresample=48000", "-ac", "1", "-c:a", "pcm_s16le", "-f", "wav", output]);105 return (await probe(output)).duration;106}107108async function synthesizeSay(o: { text: string; voiceId: string; language: Lang; speed: number; output: string }, tmp: string): Promise<{ seconds: number; voice: string }> {109 const voices = await sayVoices();110 if (!voices.length) throw new Error("`say` indisponible sur ce système");111 const voice = pickSayVoice(voices, o.voiceId, o.language);112 if (!voice) throw new Error(`Aucune voix système ${o.language} installée pour \`say\``);113 const textFile = path.join(tmp, "text.txt"), aiff = path.join(tmp, "say.aiff");114 await writeFile(textFile, o.text);115 const r = await exec("/usr/bin/say", ["-v", voice, "-r", String(Math.round(175 * Math.min(2, Math.max(0.5, o.speed)))), "-o", aiff, "-f", textFile], 600_000);116 if (r.code !== 0 || !existsSync(aiff)) throw new Error(`say a échoué : ${r.stderr.trim() || `code ${r.code}`}`);117 return { seconds: await toWav48k(aiff, o.output), voice };118}119120/** Style de narration demandé au modèle vocal d'OpenAI (champ `instructions`). */121const OPENAI_STYLE: Record<Lang, string> = {122 fr: "Narration de documentaire en français, voix posée, chaleureuse et naturelle, diction claire, rythme vivant sans précipitation, légères respirations entre les phrases. Accent français neutre.",123 en: "Documentary narration, calm, warm and natural voice, clear diction, lively but unhurried pace, slight breaths between sentences.",124};125/** Synthèse vocale OpenAI (`/v1/audio/speech`). Voix entièrement synthétiques du catalogue d'OpenAI ; aucune imitation de personne réelle. */126async function openaiSpeech(o: { text: string; voice: string; language: Lang; speed: number; output: string }) {127 const key = process.env.OPENAI_API_KEY;128 if (!key) throw new Error("Voix OpenAI demandée mais OPENAI_API_KEY n'est pas configurée");129 let last = "";130 for (let attempt = 0; attempt < 3; attempt++) {131 if (attempt) await new Promise((r) => setTimeout(r, 1500 * attempt));132 try {133 const res = await fetch("https://api.openai.com/v1/audio/speech", {134 method: "POST", headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" }, signal: AbortSignal.timeout(120_000),135 body: JSON.stringify({ model: process.env.OPENAI_TTS_MODEL ?? "gpt-4o-mini-tts", voice: o.voice, input: o.text, instructions: OPENAI_STYLE[o.language], response_format: "wav", speed: Math.min(1.5, Math.max(0.7, o.speed)) }),136 });137 if (res.ok) { await writeFile(o.output, Buffer.from(await res.arrayBuffer())); return; }138 last = `HTTP ${res.status} : ${(await res.text()).slice(0, 200).replace(/sk-[A-Za-z0-9_-]{10,}/g, "[clé]")}`;139 if (res.status < 500 && res.status !== 429) break;140 } catch (e) { last = (e as Error).message; }141 }142 throw new Error(`Synthèse vocale OpenAI en échec : ${last}`);143}144145/**146 * Synthèse vocale → WAV 48 kHz mono PCM 16 bits.147 * `provider` : `openai` (voix du catalogue OpenAI), `kokoro` (local) ou `say` (voix système macOS). Si Kokoro échoue (paquet ou modèle148 * absent), repli automatique sur `say`, signalé dans `provider` / `fallbackReason`. Tout autre fournisseur est refusé par une erreur explicite.149 * `words` : mots horodatés obtenus en retranscrivant l'audio produit avec Whisper, pour TOUS les fournisseurs (les mots sont ceux que150 * Whisper entend : nombres et sigles peuvent être normalisés) ; null si Whisper n'est pas disponible ou échoue (jamais fatal).151 * `aligned` : les mots du SCRIPT calés sur ces temps (`alignScript`, fenêtre [0, seconds]) ; null quand `words` est null — rien n'est152 * fabriqué sans mesure.153 */154export async function synthesize(o: { text: string; provider: string; voiceId: string; language: Lang; speed: number; output: string /* .wav */ }): Promise<{ seconds: number; words: WordTiming[] | null; aligned: AlignedWord[] | null; provider: string; voice: string; fallbackReason?: string; computeSeconds: number }> {155 const text = o.text.replace(/\s+/g, " ").trim();156 if (!text) throw new Error("synthesize : texte vide");157 if (o.provider !== "kokoro" && o.provider !== "say" && o.provider !== "openai") throw new Error(`synthesize : fournisseur « ${o.provider} » non pris en charge (openai, kokoro, say)`);158 const t0 = Date.now();159 const tmp = await mkdtemp(path.join(os.tmpdir(), "spclip-tts-"));160 try {161 let seconds = 0, provider = o.provider, voice = o.voiceId, fallbackReason: string | undefined;162 if (o.provider === "openai") {163 const raw = path.join(tmp, "openai.wav");164 await openaiSpeech({ text, voice: o.voiceId, language: o.language, speed: o.speed || 1, output: raw });165 seconds = await toWav48k(raw, o.output);166 } else if (o.provider === "kokoro") {167 try {168 const voiceId = [...KOKORO_VOICES.fr, ...KOKORO_VOICES.en].includes(o.voiceId) || /^[a-z]{2}_[a-z0-9]+$/.test(o.voiceId) ? o.voiceId : KOKORO_VOICES[o.language][0];169 const textFile = path.join(tmp, "text.txt"), raw = path.join(tmp, "kokoro.wav");170 await writeFile(textFile, text);171 const r = await runPy<{ ok: boolean; error?: string; seconds?: number }>("tts.py", ["--text-file", textFile, "--voice", voiceId, "--lang", o.language, "--speed", String(o.speed || 1), "--out", raw]);172 if (!r.ok) throw new Error(r.error ?? "échec inconnu");173 seconds = await toWav48k(raw, o.output);174 voice = voiceId;175 } catch (e) {176 fallbackReason = `Kokoro indisponible : ${(e as Error).message}`;177 try {178 const s = await synthesizeSay({ ...o, text }, tmp);179 seconds = s.seconds; voice = s.voice; provider = "say";180 } catch (e2) { throw new Error(`${fallbackReason} ; repli say impossible : ${(e2 as Error).message}`); }181 }182 } else {183 const s = await synthesizeSay({ ...o, text }, tmp);184 seconds = s.seconds; voice = s.voice;185 }186 if (!(seconds > 0.05)) throw new Error("synthesize : audio vide");187 let words: WordTiming[] | null = null;188 let aligned: AlignedWord[] | null = null;189 try {190 const tr = await transcribe({ file: o.output, language: o.language });191 const all = tr.segments.flatMap((s) => s.words ?? []);192 words = all.length ? all : null;193 aligned = words ? alignScript(text, words, { start: 0, end: seconds }) : null;194 } catch { words = null; aligned = null; }195 return { seconds, words, aligned, provider, voice, ...(fallbackReason ? { fallbackReason } : {}), computeSeconds: (Date.now() - t0) / 1000 };196 } finally {197 await rm(tmp, { recursive: true, force: true });198 }199}200201/**202 * Transcription Whisper locale avec horodatage par mot (médias importés, alignement de la narration).203 * La piste audio est d'abord extraite en WAV 16 kHz mono ; erreur explicite si le fichier n'a pas de piste audio ou si Whisper manque.204 */205export async function transcribe(o: { file: string; language?: Lang }): Promise<{ language: string; text: string; segments: { start: number; end: number; text: string; words?: WordTiming[] }[]; model: string; computeSeconds?: number }> {206 const p = await probe(o.file);207 if (!p.hasAudio) throw new Error("transcribe : le fichier n'a pas de piste audio");208 const tmp = await mkdtemp(path.join(os.tmpdir(), "spclip-stt-"));209 try {210 const wav = path.join(tmp, "audio.wav");211 await ff(["-i", o.file, "-vn", "-map", "0:a:0", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", wav], 1_800_000);212 const r = await runPy<{ ok: boolean; error?: string; language: string; text: string; segments: { start: number; end: number; text: string; words?: WordTiming[] }[]; model: string; computeSeconds: number }>(213 "transcribe.py", ["--file", wav, ...(o.language ? ["--lang", o.language] : [])], Math.max(FIRST_RUN_TIMEOUT, p.duration * 2000));214 if (!r.ok) throw new Error(`transcribe : ${r.error ?? "échec inconnu"}`);215 return { language: r.language, text: r.text, segments: r.segments, model: r.model, computeSeconds: r.computeSeconds };216 } finally {217 await rm(tmp, { recursive: true, force: true });218 }219}220