Add multimodality example with audio

patterns/multimodality/multimodality-openai/README.md

@@ -33,6 +33,8 @@ Run the application.
 > [!NOTE]
 > These examples use the [httpie](https://httpie.io) CLI to send HTTP requests.
 
+### Image
+
 Call the application that will use a chat model to answer your question.
 
 ```shell
@@ -50,3 +52,11 @@ The image can also be fetched from a URL.
 ```shell
 http :8080/chat/image/url question=="What's in the picture? Answer in one sentence" -b
 ```
+
+### Audio
+
+Call the application that will use a chat model to answer your question.
+
+```shell
+http :8080/chat/audio/file question=="What is this recording about? Give a short answer" -b
+```

patterns/multimodality/multimodality-openai/src/main/java/com/thomasvitale/ai/spring/ChatController.java

@@ -2,6 +2,8 @@
 
 import org.springframework.ai.chat.client.ChatClient;
 import org.springframework.ai.model.Media;
+import org.springframework.ai.openai.OpenAiChatOptions;
+import org.springframework.ai.openai.api.OpenAiApi;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
 import org.springframework.util.MimeTypeUtils;
@@ -19,6 +21,9 @@ class ChatController {
 
     private final ChatClient chatClient;
 
+    @Value("classpath:speech.mp3")
+    private Resource audio;
+
     @Value("classpath:tabby-cat.png")
     private Resource image;
 
@@ -50,4 +55,18 @@ String chatImageUrl(String question) throws MalformedURLException {
                 .content();
     }
 
+    @GetMapping("/chat/audio/file")
+    String chatAudioFile(String question) {
+        return chatClient.prompt()
+                .user(userSpec -> userSpec
+                        .text(question)
+                        .media(MimeTypeUtils.parseMimeType("audio/mp3"), audio)
+                )
+                .options(OpenAiChatOptions.builder()
+                        .withModel(OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue())
+                        .build())
+                .call()
+                .content();
+    }
+
 }

patterns/multimodality/multimodality-openai/src/main/java/com/thomasvitale/ai/spring/model/ChatModelController.java

@@ -4,6 +4,8 @@
 import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.ai.chat.prompt.Prompt;
 import org.springframework.ai.model.Media;
+import org.springframework.ai.openai.OpenAiChatOptions;
+import org.springframework.ai.openai.api.OpenAiApi;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
 import org.springframework.util.MimeTypeUtils;
@@ -23,6 +25,9 @@ class ChatModelController {
 
     private final ChatModel chatModel;
 
+    @Value("classpath:speech.mp3")
+    private Resource audio;
+
     @Value("classpath:tabby-cat.png")
     private Resource image;
 
@@ -49,4 +54,14 @@ String chatImageUrl(String question) throws MalformedURLException {
         return chatResponse.getResult().getOutput().getContent();
     }
 
+    @GetMapping("/chat/audio/file")
+    String chatAudioFile(String question) {
+        var userMessage = new UserMessage(question, new Media(MimeTypeUtils.parseMimeType("audio/mp3"), audio));
+        var prompt = new Prompt(userMessage, OpenAiChatOptions.builder()
+                .withModel(OpenAiApi.ChatModel.GPT_4_O_AUDIO_PREVIEW.getValue())
+                .build());
+        var chatResponse = chatModel.call(prompt);
+        return chatResponse.getResult().getOutput().getContent();
+    }
+
 }