SPB Git

Repository Public

speech.ts 14.5 KB 220 lines typescript

1import { execFile } from "node:child_process";2import { existsSync } from "node:fs";3import { mkdtemp, rm, writeFile } from "node:fs/promises";4import os from "node:os";5import path from "node:path";6import type { Lang, WordTiming } from "../lib/contracts";7import { env } from "../lib/env";8import { alignScript, type AlignedWord } from "./align";9import { ff, probe } from "./ffmpeg";1011/**12 * Voix et transcription locales.13 *  - TTS : Kokoro v1.0 par `kokoro-onnx` (py/tts.py ; voix `ff_siwis` en français, `af_heart` / `am_michael` en anglais) ; repli `say` (macOS).14 *  - Transcription et alignement par mot : Whisper large-v3-turbo par `mlx-whisper` (py/transcribe.py).15 * Les scripts Python sont lancés par `uv run --project <env.pyDir> python …` et répondent en JSON sur stdout.16 * Premier lancement sur un nœud : uv crée l'environnement (≈ 1 Go de paquets) et les modèles sont téléchargés17 * (Kokoro ≈ 340 Mo dans ~/.cache/sp-clip/kokoro, Whisper ≈ 1,6 Go dans le cache Hugging Face) : délais jusqu'à 30 min tolérés.18 */1920const PATHS = `${os.homedir()}/.local/bin:/opt/homebrew/bin:/usr/local/bin:${process.env.PATH ?? ""}`;21const FIRST_RUN_TIMEOUT = 30 * 60_000;2223export const KOKORO_VOICES: Record<Lang, string[]> = { fr: ["ff_siwis"], en: ["af_heart", "am_michael"] };24const SAY_VOICES: Record<Lang, string[]> = { fr: ["Amélie", "Thomas", "Audrey", "Aurélie", "Jacques", "Flo", "Eddy"], en: ["Samantha", "Alex", "Daniel", "Karen", "Moira", "Tessa", "Flo", "Eddy", "Fred"] };2526function exec(bin: string, args: string[], timeoutMs: number): Promise<{ stdout: string; stderr: string; code: number }> {27  return new Promise((resolve, reject) => {28    execFile(bin, args, { env: { ...process.env, PATH: PATHS, PYTHONUNBUFFERED: "1", HF_HUB_DISABLE_PROGRESS_BARS: "1" }, maxBuffer: 256 * 1024 * 1024, timeout: timeoutMs }, (err, stdout, stderr) => {29      if (err && (err as NodeJS.ErrnoException).code === "ENOENT") return reject(new Error(`${bin} introuvable dans le PATH`));30      if (err && err.killed) return reject(new Error(`${bin} : délai dépassé (${Math.round(timeoutMs / 1000)} s)`));31      resolve({ stdout, stderr, code: err ? (typeof err.code === "number" ? err.code : 1) : 0 });32    });33  });34}3536/** Lance un script de py/ et renvoie son JSON. Erreur explicite si uv, l'environnement ou le script échouent. */37async function runPy<T extends { ok: boolean; error?: string }>(script: string, args: string[], timeoutMs = FIRST_RUN_TIMEOUT): Promise<T> {38  const dir = env.pyDir;39  if (!existsSync(path.join(dir, script))) throw new Error(`Script Python manquant : ${path.join(dir, script)} (CLIP_PY_DIR)`);40  const r = await exec("uv", ["run", "--quiet", "--project", dir, "python", path.join(dir, script), ...args], timeoutMs);41  const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();42  let parsed: T | null = null;43  if (line) { try { parsed = JSON.parse(line) as T; } catch { parsed = null; } }44  if (!parsed) throw new Error(`${script} : pas de réponse JSON (code ${r.code}) — ${r.stderr.trim().split("\n").slice(-3).join(" | ") || "aucune sortie"}`);45  return parsed;46}4748/** Voix installées : nom complet tel que listé par `say -v ?` (ex. « Amélie (Français (Canada)) ») et langue (ex. fr_CA). */49async function sayVoices(): Promise<{ name: string; locale: string }[]> {50  if (process.platform !== "darwin" || !existsSync("/usr/bin/say")) return [];51  try {52    const { stdout } = await exec("/usr/bin/say", ["-v", "?"], 30_000);53    const seen = new Set<string>();54    return stdout.split("\n").flatMap((l) => {55      const m = l.match(/^(.+?)\s+([a-z]{2,3}_[A-Za-z0-9]+)\s+#/);56      if (!m || seen.has(m[1].trim())) return [];57      seen.add(m[1].trim());58      return [{ name: m[1].trim(), locale: m[2] }];59    });60  } catch { return []; }61}62/** Choisit une voix `say` : celle demandée si elle existe, sinon la première voix préférée de la langue, sinon n'importe quelle voix de la langue. */63function pickSayVoice(voices: { name: string; locale: string }[], voiceId: string, language: Lang): string | undefined {64  const base = (n: string) => n.replace(/\s*\(.*$/, "");65  const asked = voiceId.trim() && voices.find((v) => v.name === voiceId || base(v.name) === voiceId);66  if (asked) return asked.name;67  const inLang = voices.filter((v) => v.locale.startsWith(`${language}_`));68  for (const pref of SAY_VOICES[language]) { const v = inLang.find((x) => base(x.name) === pref); if (v) return v.name; }69  return inLang[0]?.name;70}7172let capsCache: { at: number; value: Awaited<ReturnType<typeof speechCapabilities>> } | null = null;7374/** Rapporte ce qui est réellement installé sur ce nœud (paquets importables, modèles déjà téléchargés, voix `say`). */75export async function speechCapabilities(): Promise<{ kokoro: boolean; whisper: boolean; say: boolean; detail: Record<string, string> }> {76  if (capsCache && Date.now() - capsCache.at < 60_000) return capsCache.value;77  const detail: Record<string, string> = { pyDir: env.pyDir };78  let kokoro = false, whisper = false;79  try {80    const uv = await exec("uv", ["--version"], 15_000);81    detail.uv = uv.stdout.trim() || "uv présent";82    try {83      const k = await runPy<{ ok: boolean; error?: string; package?: string; models?: boolean; cache?: string }>("tts.py", ["--check"]);84      kokoro = k.ok;85      detail.kokoro = k.ok ? `${k.package} ; modèles ${k.models ? "présents" : "à télécharger au premier usage (≈ 340 Mo)"} dans ${k.cache} ; voix ${[...KOKORO_VOICES.fr, ...KOKORO_VOICES.en].join(", ")}` : `indisponible : ${k.error}`;86    } catch (e) { detail.kokoro = `indisponible : ${(e as Error).message}`; }87    try {88      const w = await runPy<{ ok: boolean; error?: string; package?: string; model?: string; modelCached?: boolean }>("transcribe.py", ["--check"]);89      whisper = w.ok;90      detail.whisper = w.ok ? `${w.package} ; modèle ${w.model} ${w.modelCached ? "en cache" : "à télécharger au premier usage (≈ 1,6 Go)"}` : `indisponible : ${w.error}`;91    } catch (e) { detail.whisper = `indisponible : ${(e as Error).message}`; }92  } catch (e) {93    detail.uv = `indisponible : ${(e as Error).message}`;94    detail.kokoro = detail.whisper = "indisponible : uv requis";95  }96  const voices = await sayVoices();97  detail.say = voices.length ? `${voices.length} voix système (fr : ${pickSayVoice(voices, "", "fr") ?? "aucune"} ; en : ${pickSayVoice(voices, "", "en") ?? "aucune"})` : "indisponible (macOS requis)";98  const value = { kokoro, whisper, say: voices.length > 0, detail };99  capsCache = { at: Date.now(), value };100  return value;101}102103async function toWav48k(input: string, output: string): Promise<number> {104  await ff(["-i", input, "-vn", "-af", "aresample=48000", "-ac", "1", "-c:a", "pcm_s16le", "-f", "wav", output]);105  return (await probe(output)).duration;106}107108async function synthesizeSay(o: { text: string; voiceId: string; language: Lang; speed: number; output: string }, tmp: string): Promise<{ seconds: number; voice: string }> {109  const voices = await sayVoices();110  if (!voices.length) throw new Error("`say` indisponible sur ce système");111  const voice = pickSayVoice(voices, o.voiceId, o.language);112  if (!voice) throw new Error(`Aucune voix système ${o.language} installée pour \`say\``);113  const textFile = path.join(tmp, "text.txt"), aiff = path.join(tmp, "say.aiff");114  await writeFile(textFile, o.text);115  const r = await exec("/usr/bin/say", ["-v", voice, "-r", String(Math.round(175 * Math.min(2, Math.max(0.5, o.speed)))), "-o", aiff, "-f", textFile], 600_000);116  if (r.code !== 0 || !existsSync(aiff)) throw new Error(`say a échoué : ${r.stderr.trim() || `code ${r.code}`}`);117  return { seconds: await toWav48k(aiff, o.output), voice };118}119120/** Style de narration demandé au modèle vocal d'OpenAI (champ `instructions`). */121const OPENAI_STYLE: Record<Lang, string> = {122  fr: "Narration de documentaire en français, voix posée, chaleureuse et naturelle, diction claire, rythme vivant sans précipitation, légères respirations entre les phrases. Accent français neutre.",123  en: "Documentary narration, calm, warm and natural voice, clear diction, lively but unhurried pace, slight breaths between sentences.",124};125/** Synthèse vocale OpenAI (`/v1/audio/speech`). Voix entièrement synthétiques du catalogue d'OpenAI ; aucune imitation de personne réelle. */126async function openaiSpeech(o: { text: string; voice: string; language: Lang; speed: number; output: string }) {127  const key = process.env.OPENAI_API_KEY;128  if (!key) throw new Error("Voix OpenAI demandée mais OPENAI_API_KEY n'est pas configurée");129  let last = "";130  for (let attempt = 0; attempt < 3; attempt++) {131    if (attempt) await new Promise((r) => setTimeout(r, 1500 * attempt));132    try {133      const res = await fetch("https://api.openai.com/v1/audio/speech", {134        method: "POST", headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" }, signal: AbortSignal.timeout(120_000),135        body: JSON.stringify({ model: process.env.OPENAI_TTS_MODEL ?? "gpt-4o-mini-tts", voice: o.voice, input: o.text, instructions: OPENAI_STYLE[o.language], response_format: "wav", speed: Math.min(1.5, Math.max(0.7, o.speed)) }),136      });137      if (res.ok) { await writeFile(o.output, Buffer.from(await res.arrayBuffer())); return; }138      last = `HTTP ${res.status} : ${(await res.text()).slice(0, 200).replace(/sk-[A-Za-z0-9_-]{10,}/g, "[clé]")}`;139      if (res.status < 500 && res.status !== 429) break;140    } catch (e) { last = (e as Error).message; }141  }142  throw new Error(`Synthèse vocale OpenAI en échec : ${last}`);143}144145/**146 * Synthèse vocale → WAV 48 kHz mono PCM 16 bits.147 * `provider` : `openai` (voix du catalogue OpenAI), `kokoro` (local) ou `say` (voix système macOS). Si Kokoro échoue (paquet ou modèle148 * absent), repli automatique sur `say`, signalé dans `provider` / `fallbackReason`. Tout autre fournisseur est refusé par une erreur explicite.149 * `words` : mots horodatés obtenus en retranscrivant l'audio produit avec Whisper, pour TOUS les fournisseurs (les mots sont ceux que150 * Whisper entend : nombres et sigles peuvent être normalisés) ; null si Whisper n'est pas disponible ou échoue (jamais fatal).151 * `aligned` : les mots du SCRIPT calés sur ces temps (`alignScript`, fenêtre [0, seconds]) ; null quand `words` est null — rien n'est152 * fabriqué sans mesure.153 */154export async function synthesize(o: { text: string; provider: string; voiceId: string; language: Lang; speed: number; output: string /* .wav */ }): Promise<{ seconds: number; words: WordTiming[] | null; aligned: AlignedWord[] | null; provider: string; voice: string; fallbackReason?: string; computeSeconds: number }> {155  const text = o.text.replace(/\s+/g, " ").trim();156  if (!text) throw new Error("synthesize : texte vide");157  if (o.provider !== "kokoro" && o.provider !== "say" && o.provider !== "openai") throw new Error(`synthesize : fournisseur « ${o.provider} » non pris en charge (openai, kokoro, say)`);158  const t0 = Date.now();159  const tmp = await mkdtemp(path.join(os.tmpdir(), "spclip-tts-"));160  try {161    let seconds = 0, provider = o.provider, voice = o.voiceId, fallbackReason: string | undefined;162    if (o.provider === "openai") {163      const raw = path.join(tmp, "openai.wav");164      await openaiSpeech({ text, voice: o.voiceId, language: o.language, speed: o.speed || 1, output: raw });165      seconds = await toWav48k(raw, o.output);166    } else if (o.provider === "kokoro") {167      try {168        const voiceId = [...KOKORO_VOICES.fr, ...KOKORO_VOICES.en].includes(o.voiceId) || /^[a-z]{2}_[a-z0-9]+$/.test(o.voiceId) ? o.voiceId : KOKORO_VOICES[o.language][0];169        const textFile = path.join(tmp, "text.txt"), raw = path.join(tmp, "kokoro.wav");170        await writeFile(textFile, text);171        const r = await runPy<{ ok: boolean; error?: string; seconds?: number }>("tts.py", ["--text-file", textFile, "--voice", voiceId, "--lang", o.language, "--speed", String(o.speed || 1), "--out", raw]);172        if (!r.ok) throw new Error(r.error ?? "échec inconnu");173        seconds = await toWav48k(raw, o.output);174        voice = voiceId;175      } catch (e) {176        fallbackReason = `Kokoro indisponible : ${(e as Error).message}`;177        try {178          const s = await synthesizeSay({ ...o, text }, tmp);179          seconds = s.seconds; voice = s.voice; provider = "say";180        } catch (e2) { throw new Error(`${fallbackReason} ; repli say impossible : ${(e2 as Error).message}`); }181      }182    } else {183      const s = await synthesizeSay({ ...o, text }, tmp);184      seconds = s.seconds; voice = s.voice;185    }186    if (!(seconds > 0.05)) throw new Error("synthesize : audio vide");187    let words: WordTiming[] | null = null;188    let aligned: AlignedWord[] | null = null;189    try {190      const tr = await transcribe({ file: o.output, language: o.language });191      const all = tr.segments.flatMap((s) => s.words ?? []);192      words = all.length ? all : null;193      aligned = words ? alignScript(text, words, { start: 0, end: seconds }) : null;194    } catch { words = null; aligned = null; }195    return { seconds, words, aligned, provider, voice, ...(fallbackReason ? { fallbackReason } : {}), computeSeconds: (Date.now() - t0) / 1000 };196  } finally {197    await rm(tmp, { recursive: true, force: true });198  }199}200201/**202 * Transcription Whisper locale avec horodatage par mot (médias importés, alignement de la narration).203 * La piste audio est d'abord extraite en WAV 16 kHz mono ; erreur explicite si le fichier n'a pas de piste audio ou si Whisper manque.204 */205export async function transcribe(o: { file: string; language?: Lang }): Promise<{ language: string; text: string; segments: { start: number; end: number; text: string; words?: WordTiming[] }[]; model: string; computeSeconds?: number }> {206  const p = await probe(o.file);207  if (!p.hasAudio) throw new Error("transcribe : le fichier n'a pas de piste audio");208  const tmp = await mkdtemp(path.join(os.tmpdir(), "spclip-stt-"));209  try {210    const wav = path.join(tmp, "audio.wav");211    await ff(["-i", o.file, "-vn", "-map", "0:a:0", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", wav], 1_800_000);212    const r = await runPy<{ ok: boolean; error?: string; language: string; text: string; segments: { start: number; end: number; text: string; words?: WordTiming[] }[]; model: string; computeSeconds: number }>(213      "transcribe.py", ["--file", wav, ...(o.language ? ["--lang", o.language] : [])], Math.max(FIRST_RUN_TIMEOUT, p.duration * 2000));214    if (!r.ok) throw new Error(`transcribe : ${r.error ?? "échec inconnu"}`);215    return { language: r.language, text: r.text, segments: r.segments, model: r.model, computeSeconds: r.computeSeconds };216  } finally {217    await rm(tmp, { recursive: true, force: true });218  }219}220
Files main
Menu

Theme