Upgrade the Moderation API with the new multimodal moderation model

src/main/java/io/github/stefanbratanov/jvm/openai/Moderation.java

@@ -6,13 +6,19 @@
 /** Represents if a given text input is potentially harmful. */
 public record Moderation(String id, String model, List<Result> results) {
 
-  public record Result(boolean flagged, Categories categories, CategoryScores categoryScores) {
+  public record Result(
+      boolean flagged,
+      Categories categories,
+      CategoryScores categoryScores,
+      CategoryAppliedInputTypes categoryAppliedInputTypes) {
 
     public record Categories(
         boolean hate,
         @JsonProperty("hate/threatening") boolean hateThreatening,
         boolean harassment,
         @JsonProperty("harassment/threatening") boolean harassmentThreatening,
+        boolean illicit,
+        @JsonProperty("illicit/violent") boolean illicitViolent,
         @JsonProperty("self-harm") boolean selfHarm,
         @JsonProperty("self-harm/intent") boolean selfHarmIntent,
         @JsonProperty("self-harm/instructions") boolean selfHarmInstructions,
@@ -26,12 +32,29 @@ public record CategoryScores(
         @JsonProperty("hate/threatening") Double hateThreatening,
         Double harassment,
         @JsonProperty("harassment/threatening") Double harassmentThreatening,
+        Double illicit,
+        @JsonProperty("illicit/violent") Double illicitViolent,
         @JsonProperty("self-harm") Double selfHarm,
         @JsonProperty("self-harm/intent") Double selfHarmIntent,
         @JsonProperty("self-harm/instructions") Double selfHarmInstructions,
         Double sexual,
         @JsonProperty("sexual/minors") Double sexualMinors,
         Double violence,
         @JsonProperty("violence/graphic") Double violenceGraphic) {}
+
+    public record CategoryAppliedInputTypes(
+        List<String> hate,
+        @JsonProperty("hate/threatening") List<String> hateThreatening,
+        List<String> harassment,
+        @JsonProperty("harassment/threatening") List<String> harassmentThreatening,
+        List<String> illicit,
+        @JsonProperty("illicit/violent") List<String> illicitViolent,
+        @JsonProperty("self-harm") List<String> selfHarm,
+        @JsonProperty("self-harm/intent") List<String> selfHarmIntent,
+        @JsonProperty("self-harm/instructions") List<String> selfHarmInstructions,
+        List<String> sexual,
+        @JsonProperty("sexual/minors") List<String> sexualMinors,
+        List<String> violence,
+        @JsonProperty("violence/graphic") List<String> violenceGraphic) {}
   }
 }

src/main/java/io/github/stefanbratanov/jvm/openai/ModerationRequest.java

@@ -1,59 +1,106 @@
 package io.github.stefanbratanov.jvm.openai;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.ImageUrlInput;
+import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.ImageUrlInput.ImageUrl;
+import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.TextInput;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
 
-public record ModerationRequest(List<String> input, Optional<String> model) {
+public record ModerationRequest(List<Object> input, Optional<String> model) {
 
   public static Builder newBuilder() {
     return new Builder();
   }
 
   public static class Builder {
 
-    private final List<String> input = new LinkedList<>();
+    private final List<Object> input = new LinkedList<>();
 
     private Optional<String> model = Optional.empty();
 
     /**
-     * @param input input to append to the list of input texts to classify
+     * @param input a string of text to append to the list of inputs to classify for moderation
      */
     public Builder input(String input) {
       this.input.add(input);
       return this;
     }
 
     /**
-     * @param inputs inputs to append to the list of input texts to classify
+     * @param inputs an array of strings to append to the list of inputs to classify for moderation
      */
     public Builder inputs(List<String> inputs) {
       input.addAll(inputs);
       return this;
     }
 
     /**
-     * @param model Two content moderations models are available: text-moderation-stable and
-     *     text-moderation-latest.
-     *     <p>The default is text-moderation-latest which will be automatically upgraded over time.
-     *     This ensures you are always using our most accurate model. If you use
-     *     text-moderation-stable, we will provide advanced notice before updating the model.
-     *     Accuracy of text-moderation-stable may be slightly lower than for text-moderation-latest.
+     * @param input multi-modal input to append to the list of inputs to classify for moderation
+     */
+    public Builder multiModalInput(MultiModalInput input) {
+      this.input.add(input);
+      return this;
+    }
+
+    /**
+     * @param inputs an array of multi-modal inputs to append to the list of inputs to classify for
+     *     moderation
+     */
+    public Builder multiModalInputs(List<MultiModalInput> inputs) {
+      this.input.addAll(inputs);
+      return this;
+    }
+
+    /**
+     * @param model The content moderation model you would like to use.
      */
     public Builder model(String model) {
       this.model = Optional.of(model);
       return this;
     }
 
     /**
-     * @param model Two content moderations models are available: {@link
-     *     OpenAIModel#TEXT_MODERATION_LATEST} and {@link OpenAIModel#TEXT_MODERATION_STABLE}.
+     * @param model The content moderation {@link OpenAIModel} you would like to use.
      */
     public Builder model(OpenAIModel model) {
       this.model = Optional.of(model.getId());
       return this;
     }
 
+    public sealed interface MultiModalInput permits ImageUrlInput, TextInput {
+
+      @JsonProperty(access = JsonProperty.Access.READ_ONLY)
+      String type();
+
+      record ImageUrlInput(ImageUrl imageUrl) implements MultiModalInput {
+
+        @Override
+        public String type() {
+          return "image_url";
+        }
+
+        public record ImageUrl(String url) {}
+      }
+
+      record TextInput(String text) implements MultiModalInput {
+
+        @Override
+        public String type() {
+          return "text";
+        }
+      }
+
+      static ImageUrlInput imageUrl(ImageUrl imageUrl) {
+        return new ImageUrlInput(imageUrl);
+      }
+
+      static TextInput text(String text) {
+        return new TextInput(text);
+      }
+    }
+
     public ModerationRequest build() {
       return new ModerationRequest(List.copyOf(input), model);
     }

src/main/java/io/github/stefanbratanov/jvm/openai/ModerationsClient.java

@@ -8,7 +8,7 @@
 import java.util.Optional;
 
 /**
- * Given some input text, outputs if the model classifies it as potentially harmful across several
+ * Given text and/or image inputs, classifies if those inputs are potentially harmful across several
  * categories.
  *
  * <p>Based on <a href="https://platform.openai.com/docs/api-reference/moderations">Moderations</a>
@@ -27,7 +27,7 @@ public final class ModerationsClient extends OpenAIClient {
   }
 
   /**
-   * Classifies if text is potentially harmful.
+   * Classifies if text and/or image inputs are potentially harmful.
    *
    * @throws OpenAIException in case of API errors
    */

src/main/java/io/github/stefanbratanov/jvm/openai/OpenAIModel.java

@@ -53,6 +53,7 @@ public enum OpenAIModel {
   TEXT_EMBEDDING_ADA_002("text-embedding-ada-002"),
 
   // Moderation (https://platform.openai.com/docs/models/moderation)
+  OMNI_MODERATION_LATEST("omni-moderation-latest"),
   TEXT_MODERATION_LATEST("text-moderation-latest"),
   TEXT_MODERATION_STABLE("text-moderation-stable");
 

src/test/java/io/github/stefanbratanov/jvm/openai/TestDataUtil.java

@@ -25,6 +25,8 @@
 import io.github.stefanbratanov.jvm.openai.CompletionUsage.CompletionTokensDetails;
 import io.github.stefanbratanov.jvm.openai.CreateChatCompletionRequest.StreamOptions;
 import io.github.stefanbratanov.jvm.openai.FineTuningJobIntegration.Wandb;
+import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput;
+import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.ImageUrlInput.ImageUrl;
 import io.github.stefanbratanov.jvm.openai.ProjectApiKey.Owner;
 import io.github.stefanbratanov.jvm.openai.ProjectApiKeysClient.PaginatedProjectApiKeys;
 import io.github.stefanbratanov.jvm.openai.ProjectServiceAccountsClient.ApiKey;
@@ -343,12 +345,20 @@ public ModerationRequest randomModerationRequest() {
     ModerationRequest.Builder builder = ModerationRequest.newBuilder();
     runOne(
         () -> builder.input(randomString(10)),
-        () -> builder.inputs(listOf(randomInt(1, 5), () -> randomString(10))));
+        () -> builder.inputs(listOf(randomInt(1, 5), () -> randomString(10))),
+        () -> builder.multiModalInput(randomMultiModalInput()),
+        () -> builder.multiModalInputs(listOf(randomInt(1, 5), this::randomMultiModalInput)));
     return builder
         .model(oneOf(OpenAIModel.TEXT_MODERATION_LATEST, OpenAIModel.TEXT_MODERATION_STABLE))
         .build();
   }
 
+  public MultiModalInput randomMultiModalInput() {
+    return oneOf(
+        MultiModalInput.imageUrl(new ImageUrl("https://example.com/image.jpg")),
+        MultiModalInput.text(randomString(10)));
+  }
+
   public Moderation randomModeration() {
     return new Moderation(
         randomString(6),
@@ -369,6 +379,8 @@ public Moderation randomModeration() {
                         randomBoolean(),
                         randomBoolean(),
                         randomBoolean(),
+                        randomBoolean(),
+                        randomBoolean(),
                         randomBoolean()),
                     new Moderation.Result.CategoryScores(
                         randomDouble(),
@@ -381,7 +393,23 @@ public Moderation randomModeration() {
                         randomDouble(),
                         randomDouble(),
                         randomDouble(),
-                        randomDouble()))));
+                        randomDouble(),
+                        randomDouble(),
+                        randomDouble()),
+                    new Moderation.Result.CategoryAppliedInputTypes(
+                        List.of("text"),
+                        List.of("text"),
+                        List.of("text"),
+                        List.of("text"),
+                        List.of("text"),
+                        List.of("text"),
+                        List.of("text", "image"),
+                        List.of("text", "image"),
+                        List.of("text", "image"),
+                        List.of("text", "image"),
+                        List.of("text"),
+                        List.of("text", "image"),
+                        List.of("text", "image")))));
   }
 
   public CreateAssistantRequest randomCreateAssistantRequest() {