Repository Public
1# Audio in Chat Completions23> For the complete documentation index, see [llms.txt](/llms.txt). Markdown versions of documentation pages are available by appending `.md` to the page URL.45If you already have a text-based LLM application with the [Chat Completions endpoint](https://developers.openai.com/api/reference/resources/chat), you may want to add audio capabilities. For example, if your chat application supports text input, you can add audio input and output: include `audio` in the `modalities` array and use an audio model, like [`gpt-audio-1.5`](https://developers.openai.com/api/docs/models/gpt-audio-1.5).67The [Responses API](https://developers.openai.com/api/reference/resources/responses) docs currently describe8 text and image inputs with text outputs. For this audio-chat pattern, use Chat9 Completions with an audio-capable model.10111213Audio output from model1415 Create a human-like audio response to a prompt1617```javascript18import { writeFileSync } from "node:fs";19import OpenAI from "openai";2021const openai = new OpenAI();2223// Generate an audio response to the given prompt24const response = await openai.chat.completions.create({25 model: "gpt-audio-1.5",26 modalities: ["text", "audio"],27 audio: { voice: "alloy", format: "wav" },28 messages: [29 {30 role: "user",31 content: "Is a golden retriever a good family dog?",32 },33 ],34 store: true,35});3637// Inspect returned data38console.log(response.choices[0]);3940// Write audio data to a file41writeFileSync(42 "dog.wav",43 Buffer.from(response.choices[0].message.audio.data, "base64"),44 { encoding: "utf-8" }45);46```4748```python49import base6450from openai import OpenAI5152client = OpenAI()5354completion = client.chat.completions.create(55 model="gpt-audio-1.5",56 modalities=["text", "audio"],57 audio={"voice": "alloy", "format": "wav"},58 messages=[{"role": "user", "content": "Is a golden retriever a good family dog?"}],59)6061print(completion.choices[0])6263wav_bytes = base64.b64decode(completion.choices[0].message.audio.data)64with open("dog.wav", "wb") as f:65 f.write(wav_bytes)66```6768```go69package main7071import (72 "context"73 "encoding/base64"74 "fmt"75 "os"7677 "github.com/openai/openai-go/v3"78)7980func main() {81 client := openai.NewClient()82 response, err := client.Chat.Completions.New(context.Background(), openai.ChatCompletionNewParams{83 Model: "gpt-audio-1.5",84 Modalities: []string{"text", "audio"},85 Audio: openai.ChatCompletionAudioParam{86 Voice: openai.ChatCompletionAudioParamVoiceUnion{OfString: openai.String("alloy")},87 Format: openai.ChatCompletionAudioParamFormatWAV,88 },89 Messages: []openai.ChatCompletionMessageParamUnion{openai.UserMessage("Is a golden retriever a good family dog?")},90 })91 if err != nil {92 panic(err)93 }94 fmt.Println(response.Choices[0])95 audio, err := base64.StdEncoding.DecodeString(response.Choices[0].Message.Audio.Data)96 if err != nil {97 panic(err)98 }99 if err := os.WriteFile("dog.wav", audio, 0o600); err != nil {100 panic(err)101 }102}103```104105```java106import com.openai.client.OpenAIClient;107import com.openai.client.okhttp.OpenAIOkHttpClient;108import com.openai.models.chat.completions.ChatCompletionAudioParam;109import com.openai.models.chat.completions.ChatCompletionCreateParams;110import java.io.IOException;111import java.nio.file.Files;112import java.nio.file.Path;113import java.util.Base64;114115ChatCompletionCreateParams params =116 ChatCompletionCreateParams.builder()117 .model("gpt-audio-1.5")118 .addUserMessage("Is a golden retriever a good family dog?")119 .addModality(ChatCompletionCreateParams.Modality.TEXT)120 .addModality(ChatCompletionCreateParams.Modality.AUDIO)121 .audio(122 ChatCompletionAudioParam.builder()123 .voice("alloy")124 .format(ChatCompletionAudioParam.Format.WAV)125 .build())126 .store(true)127 .build();128129var message = client.chat().completions().create(params).choices().get(0).message();130var audio =131 message.audio().orElseThrow(() -> new IllegalStateException("No audio output returned"));132Files.write(Path.of("dog.wav"), Base64.getDecoder().decode(audio.data()));133message.content().ifPresent(System.out::println);134```135136```csharp137using OpenAI.Chat;138#pragma warning disable OPENAI001139140string key = Environment.GetEnvironmentVariable("OPENAI_API_KEY")!;141ChatClient client = new("gpt-audio-1.5", key);142143ChatCompletionOptions options = new()144{145 ResponseModalities = ChatResponseModalities.Text | ChatResponseModalities.Audio,146 AudioOptions = new(ChatOutputAudioVoice.Alloy, ChatOutputAudioFormat.Wav),147 StoredOutputEnabled = true,148};149150ChatCompletion completion = await client.CompleteChatAsync(151 [new UserChatMessage("Is a golden retriever a good family dog?")],152 options153);154155if (completion.OutputAudio is not ChatOutputAudio audio)156{157 throw new InvalidOperationException("No audio output was returned.");158}159160Console.WriteLine(audio.Transcript);161await File.WriteAllBytesAsync("dog.wav", audio.AudioBytes.ToArray());162```163164```ruby165require "base64"166require "openai"167168client = OpenAI::Client.new169completion = client.chat.completions.create(170 model: "gpt-audio-1.5",171 messages: [172 {173 role: :user,174 content: "Is a golden retriever a good family dog?"175 }176 ],177 modalities: [:text, :audio],178 audio: {179 voice: :alloy,180 format: :wav181 },182 store: true183)184185audio = completion.choices.fetch(0).message.audio or raise "No audio returned"186File.binwrite("dog.wav", Base64.strict_decode64(audio.data))187```188189```bash190curl "https://api.openai.com/v1/chat/completions" \191 -H "Content-Type: application/json" \192 -H "Authorization: Bearer $OPENAI_API_KEY" \193 -d '{194 "model": "gpt-audio-1.5",195 "modalities": ["text", "audio"],196 "audio": { "voice": "alloy", "format": "wav" },197 "messages": [198 {199 "role": "user",200 "content": "Is a golden retriever a good family dog?"201 }202 ]203 }'204```205206 207208 209210 211Audio input to model212213 Use audio inputs for prompting a model214215```javascript216import OpenAI from "openai";217const openai = new OpenAI();218219// Fetch an audio file and convert it to a base64 string220const url = "https://cdn.openai.com/API/docs/audio/alloy.wav";221const audioResponse = await fetch(url);222const buffer = await audioResponse.arrayBuffer();223const base64str = Buffer.from(buffer).toString("base64");224225const response = await openai.chat.completions.create({226 model: "gpt-audio-1.5",227 modalities: ["text", "audio"],228 audio: { voice: "alloy", format: "wav" },229 messages: [230 {231 role: "user",232 content: [233 { type: "text", text: "What is in this recording?" },234 {235 type: "input_audio",236 input_audio: { data: base64str, format: "wav" },237 },238 ],239 },240 ],241 store: true,242});243244console.log(response.choices[0]);245```246247```python248import base64249import requests250from openai import OpenAI251252client = OpenAI()253254# Fetch the audio file and convert it to a base64 encoded string255url = "https://cdn.openai.com/API/docs/audio/alloy.wav"256response = requests.get(url)257response.raise_for_status()258wav_data = response.content259encoded_string = base64.b64encode(wav_data).decode("utf-8")260261completion = client.chat.completions.create(262 model="gpt-audio-1.5",263 modalities=["text", "audio"],264 audio={"voice": "alloy", "format": "wav"},265 messages=[266 {267 "role": "user",268 "content": [269 {"type": "text", "text": "What is in this recording?"},270 {271 "type": "input_audio",272 "input_audio": {"data": encoded_string, "format": "wav"},273 },274 ],275 },276 ],277)278279print(completion.choices[0].message)280```281282```go283package main284285import (286 "context"287 "encoding/base64"288 "fmt"289 "os"290291 "github.com/openai/openai-go/v3"292)293294func main() {295 audio, err := os.ReadFile("fixtures/audio.wav")296 if err != nil {297 panic(err)298 }299 client := openai.NewClient()300 response, err := client.Chat.Completions.New(context.Background(), openai.ChatCompletionNewParams{301 Model: "gpt-audio-1.5",302 Modalities: []string{"text", "audio"},303 Audio: openai.ChatCompletionAudioParam{304 Voice: openai.ChatCompletionAudioParamVoiceUnion{OfString: openai.String("alloy")},305 Format: openai.ChatCompletionAudioParamFormatWAV,306 },307 Messages: []openai.ChatCompletionMessageParamUnion{openai.UserMessage([]openai.ChatCompletionContentPartUnionParam{308 openai.TextContentPart("What is in this recording?"),309 openai.InputAudioContentPart(openai.ChatCompletionContentPartInputAudioInputAudioParam{310 Data: base64.StdEncoding.EncodeToString(audio),311 Format: "wav",312 }),313 })},314 })315 if err != nil {316 panic(err)317 }318 fmt.Println(response.Choices[0])319}320```321322```java323import com.openai.client.OpenAIClient;324import com.openai.client.okhttp.OpenAIOkHttpClient;325import com.openai.models.chat.completions.ChatCompletionAudioParam;326import com.openai.models.chat.completions.ChatCompletionContentPart;327import com.openai.models.chat.completions.ChatCompletionContentPartInputAudio;328import com.openai.models.chat.completions.ChatCompletionContentPartText;329import com.openai.models.chat.completions.ChatCompletionCreateParams;330import com.openai.models.chat.completions.ChatCompletionUserMessageParam;331import java.io.IOException;332import java.nio.file.Files;333import java.nio.file.Path;334import java.util.Base64;335import java.util.List;336337String encodedAudio =338 Base64.getEncoder()339 .encodeToString(340 Files.readAllBytes(Path.of(System.getenv("OPENAI_EXAMPLE_AUDIO_PATH"))));341342ChatCompletionCreateParams params =343 ChatCompletionCreateParams.builder()344 .model("gpt-audio-1.5")345 .addMessage(346 ChatCompletionUserMessageParam.builder()347 .contentOfArrayOfContentParts(348 List.of(349 ChatCompletionContentPart.ofText(350 ChatCompletionContentPartText.builder()351 .text("What is in this recording?")352 .build()),353 ChatCompletionContentPart.ofInputAudio(354 ChatCompletionContentPartInputAudio.builder()355 .inputAudio(356 ChatCompletionContentPartInputAudio.InputAudio.builder()357 .data(encodedAudio)358 .format(359 ChatCompletionContentPartInputAudio.InputAudio360 .Format.WAV)361 .build())362 .build())))363 .build())364 .addModality(ChatCompletionCreateParams.Modality.TEXT)365 .addModality(ChatCompletionCreateParams.Modality.AUDIO)366 .audio(367 ChatCompletionAudioParam.builder()368 .voice("alloy")369 .format(ChatCompletionAudioParam.Format.WAV)370 .build())371 .store(true)372 .build();373374client.chat().completions().create(params).choices().stream()375 .flatMap(choice -> choice.message().content().stream())376 .forEach(System.out::println);377```378379```csharp380using OpenAI.Chat;381#pragma warning disable OPENAI001382383string key = Environment.GetEnvironmentVariable("OPENAI_API_KEY")!;384ChatClient client = new("gpt-audio-1.5", key);385386BinaryData audio = BinaryData.FromBytes(387 await File.ReadAllBytesAsync("audio.wav")388);389UserChatMessage message = new(390 [391 ChatMessageContentPart.CreateTextPart("What is in this recording?"),392 ChatMessageContentPart.CreateInputAudioPart(393 audio,394 ChatInputAudioFormat.Wav395 ),396 ]397);398ChatCompletionOptions options = new()399{400 ResponseModalities = ChatResponseModalities.Text | ChatResponseModalities.Audio,401 AudioOptions = new(ChatOutputAudioVoice.Alloy, ChatOutputAudioFormat.Wav),402 StoredOutputEnabled = true,403};404405ChatCompletion completion = await client.CompleteChatAsync([message], options);406407if (completion.OutputAudio is not ChatOutputAudio audioOutput)408{409 throw new InvalidOperationException("No audio output was returned.");410}411412Console.WriteLine(audioOutput.Transcript);413```414415```ruby416require "base64"417require "openai"418419client = OpenAI::Client.new420audio = Base64.strict_encode64(File.binread("audio.wav"))421completion = client.chat.completions.create(422 model: "gpt-audio-1.5",423 messages: [424 {425 role: :user,426 content: [427 {428 type: :text,429 text: "What is in this recording?"430 },431 {432 type: :input_audio,433 input_audio: {434 data: audio,435 format: :wav436 }437 }438 ]439 }440 ],441 modalities: [:text, :audio],442 audio: {443 voice: :alloy,444 format: :wav445 },446 store: true447)448449puts(completion.choices.fetch(0).message.content)450```451452```bash453curl "https://api.openai.com/v1/chat/completions" \454 -H "Content-Type: application/json" \455 -H "Authorization: Bearer $OPENAI_API_KEY" \456 -d '{457 "model": "gpt-audio-1.5",458 "modalities": ["text", "audio"],459 "audio": { "voice": "alloy", "format": "wav" },460 "messages": [461 {462 "role": "user",463 "content": [464 { "type": "text", "text": "What is in this recording?" },465 { 466 "type": "input_audio", 467 "input_audio": { 468 "data": "<base64 bytes here>", 469 "format": "wav" 470 }471 }472 ]473 }474 ]475 }'476```