spb/chat-spboucher-ai

1/**2 * Modality-specific real tests: image generation, TTS, STT, embeddings, moderation (OpenAI).3 * Writes benchmarks/modality-results.json. Cheap settings only (low quality, short audio).4 */5import OpenAI from "openai";6import fs from "node:fs";78const openai = new OpenAI({ timeout: 5 * 60 * 1000, maxRetries: 1 });9const discovery = JSON.parse(fs.readFileSync("benchmarks/model-discovery.json", "utf8"));10const ids = new Set<string>(discovery.openai.map((m: any) => m.id));11const out: Record<string, any> = {};12fs.mkdirSync("benchmarks/artifacts", { recursive: true });1314async function timed<T>(id: string, kind: string, fn: () => Promise<T>, summarize: (r: T) => any) {15  const t0 = Date.now();16  try { const r = await fn(); out[id] = { kind, ok: true, latencyMs: Date.now() - t0, ...summarize(r) }; console.log("✔", id, out[id].latencyMs + "ms"); }17  catch (e: any) { out[id] = { kind, ok: false, latencyMs: Date.now() - t0, error: `${e?.status ?? ""} ${String(e?.message ?? e).slice(0, 300)}`.trim() }; console.log("✖", id, out[id].error); }18}1920// ---- image generation (one small, low-quality image per model)21for (const id of ["gpt-image-1", "gpt-image-1-mini", "gpt-image-1.5", "gpt-image-2", "gpt-image-2.5-flare", "gpt-image-2.5-sunburst", "chatgpt-image-latest"].filter(i => ids.has(i))) {22  await timed(id, "image", async () => {23    const params: any = { model: id, prompt: "A minimalist flat icon of a maple leaf, white background", n: 1, size: "1024x1024" };24    try { return await openai.images.generate({ ...params, quality: "low" }); }25    catch (e: any) { if (e?.status === 400 && /quality/i.test(e.message)) return await openai.images.generate(params); throw e; }26  }, (r: any) => { const d = r.data?.[0]; if (d?.b64_json) fs.writeFileSync(`benchmarks/artifacts/${id}.png`, Buffer.from(d.b64_json, "base64")); return { usage: r.usage, hasImage: !!(d?.b64_json || d?.url), size: r.size, quality: r.quality }; });27}2829// ---- TTS30const ttsText = "Bonjour, ceci est un test de synthèse vocale pour Chat SPBoucher AI.";31for (const id of ["tts-1", "tts-1-hd", "gpt-4o-mini-tts"].filter(i => ids.has(i))) {32  await timed(id, "tts", async () => openai.audio.speech.create({ model: id, voice: "alloy", input: ttsText, response_format: "mp3" }), () => ({}));33  if (out[id].ok) { /* re-run to save bytes */ }34}35let mp3: Buffer | null = null;36try { const r = await openai.audio.speech.create({ model: "gpt-4o-mini-tts", voice: "alloy", input: ttsText, response_format: "mp3" }); mp3 = Buffer.from(await r.arrayBuffer()); fs.writeFileSync("benchmarks/artifacts/tts-sample.mp3", mp3); } catch {}37if (!mp3) { try { const r = await openai.audio.speech.create({ model: "tts-1", voice: "alloy", input: ttsText }); mp3 = Buffer.from(await r.arrayBuffer()); fs.writeFileSync("benchmarks/artifacts/tts-sample.mp3", mp3); } catch {} }3839// ---- STT (transcribe the generated sample)40if (mp3) {41  for (const id of ["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe", "gpt-transcribe", "gpt-4o-transcribe-diarize"].filter(i => ids.has(i))) {42    await timed(id, "stt", async () => {43      const file = new File([new Uint8Array(mp3!)], "sample.mp3", { type: "audio/mpeg" });44      return openai.audio.transcriptions.create({ model: id, file, ...(id === "gpt-4o-transcribe-diarize" ? { response_format: "diarized_json" as any } : {}) });45    }, (r: any) => ({ text: (r.text ?? JSON.stringify(r)).slice(0, 200), correct: /synth[eè]se vocale/i.test(r.text ?? JSON.stringify(r)) }));46  }47}4849// ---- audio chat models (text in, text out — audio modality verified by model acceptance)50for (const id of ["gpt-audio", "gpt-audio-mini", "gpt-audio-1.5"].filter(i => ids.has(i))) {51  await timed(id, "audio-chat", async () => openai.chat.completions.create({ model: id, modalities: ["text"], messages: [{ role: "user", content: "Say 'ready' and nothing else." }] } as any), (r: any) => ({ text: r.choices?.[0]?.message?.content?.slice(0, 100), usage: r.usage }));52}5354// ---- embeddings55for (const id of ["text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002"].filter(i => ids.has(i))) {56  await timed(id, "embedding", async () => openai.embeddings.create({ model: id, input: "évaluation immobilière Gatineau" }), (r: any) => ({ dims: r.data?.[0]?.embedding?.length, usage: r.usage }));57}5859// ---- moderation60for (const id of ["omni-moderation-latest"].filter(i => ids.has(i))) {61  await timed(id, "moderation", async () => openai.moderations.create({ model: id, input: "I love Quebec winters." }), (r: any) => ({ flagged: r.results?.[0]?.flagged }));62}6364fs.writeFileSync("benchmarks/modality-results.json", JSON.stringify({ generatedAt: new Date().toISOString(), results: out }, null, 2));65console.log("done");66