/**
 * Modality-specific real tests: image generation, TTS, STT, embeddings, moderation (OpenAI).
 * Writes benchmarks/modality-results.json. Cheap settings only (low quality, short audio).
 */
import OpenAI from "openai";
import fs from "node:fs";

const openai = new OpenAI({ timeout: 5 * 60 * 1000, maxRetries: 1 });
const discovery = JSON.parse(fs.readFileSync("benchmarks/model-discovery.json", "utf8"));
const ids = new Set<string>(discovery.openai.map((m: any) => m.id));
const out: Record<string, any> = {};
fs.mkdirSync("benchmarks/artifacts", { recursive: true });

async function timed<T>(id: string, kind: string, fn: () => Promise<T>, summarize: (r: T) => any) {
  const t0 = Date.now();
  try { const r = await fn(); out[id] = { kind, ok: true, latencyMs: Date.now() - t0, ...summarize(r) }; console.log("✔", id, out[id].latencyMs + "ms"); }
  catch (e: any) { out[id] = { kind, ok: false, latencyMs: Date.now() - t0, error: `${e?.status ?? ""} ${String(e?.message ?? e).slice(0, 300)}`.trim() }; console.log("✖", id, out[id].error); }
}

// ---- image generation (one small, low-quality image per model)
for (const id of ["gpt-image-1", "gpt-image-1-mini", "gpt-image-1.5", "gpt-image-2", "gpt-image-2.5-flare", "gpt-image-2.5-sunburst", "chatgpt-image-latest"].filter(i => ids.has(i))) {
  await timed(id, "image", async () => {
    const params: any = { model: id, prompt: "A minimalist flat icon of a maple leaf, white background", n: 1, size: "1024x1024" };
    try { return await openai.images.generate({ ...params, quality: "low" }); }
    catch (e: any) { if (e?.status === 400 && /quality/i.test(e.message)) return await openai.images.generate(params); throw e; }
  }, (r: any) => { const d = r.data?.[0]; if (d?.b64_json) fs.writeFileSync(`benchmarks/artifacts/${id}.png`, Buffer.from(d.b64_json, "base64")); return { usage: r.usage, hasImage: !!(d?.b64_json || d?.url), size: r.size, quality: r.quality }; });
}

// ---- TTS
const ttsText = "Bonjour, ceci est un test de synthèse vocale pour Chat SPBoucher AI.";
for (const id of ["tts-1", "tts-1-hd", "gpt-4o-mini-tts"].filter(i => ids.has(i))) {
  await timed(id, "tts", async () => openai.audio.speech.create({ model: id, voice: "alloy", input: ttsText, response_format: "mp3" }), () => ({}));
  if (out[id].ok) { /* re-run to save bytes */ }
}
let mp3: Buffer | null = null;
try { const r = await openai.audio.speech.create({ model: "gpt-4o-mini-tts", voice: "alloy", input: ttsText, response_format: "mp3" }); mp3 = Buffer.from(await r.arrayBuffer()); fs.writeFileSync("benchmarks/artifacts/tts-sample.mp3", mp3); } catch {}
if (!mp3) { try { const r = await openai.audio.speech.create({ model: "tts-1", voice: "alloy", input: ttsText }); mp3 = Buffer.from(await r.arrayBuffer()); fs.writeFileSync("benchmarks/artifacts/tts-sample.mp3", mp3); } catch {} }

// ---- STT (transcribe the generated sample)
if (mp3) {
  for (const id of ["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe", "gpt-transcribe", "gpt-4o-transcribe-diarize"].filter(i => ids.has(i))) {
    await timed(id, "stt", async () => {
      const file = new File([new Uint8Array(mp3!)], "sample.mp3", { type: "audio/mpeg" });
      return openai.audio.transcriptions.create({ model: id, file, ...(id === "gpt-4o-transcribe-diarize" ? { response_format: "diarized_json" as any } : {}) });
    }, (r: any) => ({ text: (r.text ?? JSON.stringify(r)).slice(0, 200), correct: /synth[eè]se vocale/i.test(r.text ?? JSON.stringify(r)) }));
  }
}

// ---- audio chat models (text in, text out — audio modality verified by model acceptance)
for (const id of ["gpt-audio", "gpt-audio-mini", "gpt-audio-1.5"].filter(i => ids.has(i))) {
  await timed(id, "audio-chat", async () => openai.chat.completions.create({ model: id, modalities: ["text"], messages: [{ role: "user", content: "Say 'ready' and nothing else." }] } as any), (r: any) => ({ text: r.choices?.[0]?.message?.content?.slice(0, 100), usage: r.usage }));
}

// ---- embeddings
for (const id of ["text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002"].filter(i => ids.has(i))) {
  await timed(id, "embedding", async () => openai.embeddings.create({ model: id, input: "évaluation immobilière Gatineau" }), (r: any) => ({ dims: r.data?.[0]?.embedding?.length, usage: r.usage }));
}

// ---- moderation
for (const id of ["omni-moderation-latest"].filter(i => ids.has(i))) {
  await timed(id, "moderation", async () => openai.moderations.create({ model: id, input: "I love Quebec winters." }), (r: any) => ({ flagged: r.results?.[0]?.flagged }));
}

fs.writeFileSync("benchmarks/modality-results.json", JSON.stringify({ generatedAt: new Date().toISOString(), results: out }, null, 2));
console.log("done");