Add support for DALL-E #1358

app/components/ImageList.module.scss

@@ -0,0 +1,16 @@
+.imageGrid,
+.imageGridSingle {
+  display: inline-flex;
+  flex-wrap: wrap;
+  justify-content: flex-start;
+}
+
+.imageGrid img {
+  max-width: calc(50% - 10px);
+  margin: 5px;
+}
+
+.imageGridSingle img {
+  max-width: 100%;
+  margin: 5px;
+}

app/components/ImageList.tsx

@@ -0,0 +1,20 @@
+import { ImagesResponseDataInner } from "openai";
+import React, { FC } from "react";
+import styles from "./ImageList.module.scss";
+
+interface ImageListProps {
+  images?: ImagesResponseDataInner[];
+}
+const ImageList: FC<ImageListProps> = ({ images }) => {
+  const singleImage = images && images.length === 1;
+
+  return (
+    <div className={singleImage ? styles.imageGridSingle : styles.imageGrid}>
+      {images &&
+        images.map((image, index) => (
+          <img key={index} src={image.url} alt={`Image ${index}`} />
+        ))}
+    </div>
+  );
+};
+export default ImageList;

app/components/chat.tsx

@@ -760,6 +760,8 @@ export function Chat() {
                   )}
                   <Markdown
                     content={message.content}
+                    images={message.images}
+                    image_alt={message.image_alt}
                     loading={
                       (message.preview || message.content.length === 0) &&
                       !isUser

app/components/markdown.tsx

@@ -1,3 +1,5 @@
+import styles from "./markdown.tsx.module.scss";
+
 import ReactMarkdown from "react-markdown";
 import "katex/dist/katex.min.css";
 import RemarkMath from "remark-math";
@@ -11,7 +13,11 @@ import mermaid from "mermaid";
 
 import LoadingIcon from "../icons/three-dots.svg";
 import React from "react";
-
+import ImagePlaceHolder from "../icons/image-placeholder.svg";
+import ImageError from "../icons/image-error.svg";
+import { IMAGE_PLACEHOLDER } from "../constant";
+import { ImagesResponseDataInner } from "openai";
+import ImageList from "./ImageList";
 export function Mermaid(props: { code: string; onError: () => void }) {
   const ref = useRef<HTMLDivElement>(null);
 
@@ -119,6 +125,8 @@ export const MarkdownContent = React.memo(_MarkDownContent);
 export function Markdown(
   props: {
     content: string;
+    images?: ImagesResponseDataInner[];
+    image_alt?: string;
     loading?: boolean;
     fontSize?: number;
     parentRef: RefObject<HTMLDivElement>;
@@ -171,9 +179,24 @@ export function Markdown(
     >
       {inView.current &&
         (props.loading ? (
-          <LoadingIcon />
+          <div className={styles.loader}>
+            {props.image_alt && <MarkdownContent content={props.image_alt} />}
+            {props.image_alt &&
+              (props.image_alt === IMAGE_PLACEHOLDER ? (
+                <ImagePlaceHolder width="100px" height="100px" />
+              ) : (
+                <ImageError width="100px" height="100px" />
+              ))}
+            <LoadingIcon />
+          </div>
         ) : (
-          <MarkdownContent content={props.content} />
+          <div>
+            <MarkdownContent content={props.content} />
+            <div className={styles.content_image}>
+              {props.images && <ImageList images={props.images} />}
+              {props.image_alt && <ImageError width="100px" height="100px" />}
+            </div>
+          </div>
         ))}
     </div>
   );

app/components/markdown.tsx.module.scss

@@ -0,0 +1,10 @@
+.loader {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  height: 100%; // Adjust as necessary
+}
+.content_image {
+  text-align: center;
+}

app/constant.ts

@@ -40,3 +40,5 @@ export const NARROW_SIDEBAR_WIDTH = 100;
 export const ACCESS_CODE_PREFIX = "ak-";
 
 export const LAST_INPUT_KEY = "last-input";
+export const IMAGE_PLACEHOLDER = "Loading your image...";
+export const IMAGE_ERROR = "IMAGE_ERROR";

app/locales/cn.ts

@@ -170,7 +170,8 @@ const cn = {
   },
   Store: {
     DefaultTopic: "新的聊天",
-    BotHello: "有什么可以帮你的吗",
+    BotHello:
+      "您好！今天我能为您做些什么呢？\n 要生成图片，请使用 `/Image {关键词}。`",
     Error: "出错了，稍后重试吧",
     Prompt: {
       History: (content: string) =>

app/locales/en.ts

@@ -172,7 +172,8 @@ const en: LocaleType = {
   },
   Store: {
     DefaultTopic: "New Conversation",
-    BotHello: "Hello! How can I assist you today?",
+    BotHello:
+      "Hello! How can I assist you today?\n To generate images, use `/Image {keyword}.`",
     Error: "Something went wrong, please try again later.",
     Prompt: {
       History: (content: string) =>

app/requests.ts

@@ -8,8 +8,14 @@ import {
   useChatStore,
 } from "./store";
 import { showToast } from "./components/ui-lib";
-import { ACCESS_CODE_PREFIX } from "./constant";
-
+import { ACCESS_CODE_PREFIX, IMAGE_ERROR, IMAGE_PLACEHOLDER } from "./constant";
+import {
+  CreateImageRequest,
+  CreateImageRequestResponseFormatEnum,
+  CreateImageRequestSizeEnum,
+  ImagesResponse,
+  ImagesResponseDataInner,
+} from "openai";
 const TIME_OUT_MS = 60000;
 
 const makeRequestParam = (
@@ -144,6 +150,93 @@ export async function requestUsage() {
     subscription: total.hard_limit_usd,
   };
 }
+const makeImageRequestParam = (
+  prompt: string,
+  options?: Omit<CreateImageRequest, "prompt">,
+): CreateImageRequest => {
+  // Set default values
+  const defaultOptions: Omit<CreateImageRequest, "prompt"> = {
+    n: 4,
+    size: CreateImageRequestSizeEnum._512x512,
+    response_format: CreateImageRequestResponseFormatEnum.Url,
+    user: "default_user",
+  };
+
+  // Override default values with provided options
+  const finalOptions = { ...defaultOptions, ...options };
+
+  const request: CreateImageRequest = {
+    prompt,
+    ...finalOptions,
+  };
+
+  return request;
+};
+export async function requestImage(
+  keyword: string,
+  options?: {
+    onMessage: (
+      message: string | null,
+      image: ImagesResponseDataInner[] | null,
+      image_alt: string | null,
+      done: boolean,
+    ) => void;
+    onError: (error: Error, statusCode?: number) => void;
+    onController?: (controller: AbortController) => void;
+  },
+) {
+  if (keyword.length < 1) {
+    options?.onMessage(
+      "Please enter a keyword after `/image`",
+      null,
+      null,
+      true,
+    );
+  } else {
+    const controller = new AbortController();
+    const reqTimeoutId = setTimeout(() => controller.abort(), TIME_OUT_MS);
+    options?.onController?.(controller);
+
+    async function fetchImageAndUpdateMessage() {
+      try {
+        options?.onMessage(null, null, IMAGE_PLACEHOLDER, false);
+
+        const sanitizedMessage = keyword.replace(/[\n\r]+/g, " ");
+        const req = makeImageRequestParam(sanitizedMessage);
+
+        const res = await requestOpenaiClient("v1/images/generations")(req);
+
+        clearTimeout(reqTimeoutId);
+
+        const finish = (images: ImagesResponseDataInner[]) => {
+          options?.onMessage("Here is your images", images, null, true);
+          controller.abort();
+        };
+
+        if (res.ok) {
+          const responseData = (await res.json()) as ImagesResponse;
+          finish(responseData.data);
+        } else if (res.status === 401) {
+          console.error("Unauthorized");
+          options?.onError(new Error("Unauthorized"), res.status);
+        } else {
+          console.error("Stream Error", res.body);
+          options?.onError(new Error("Stream Error"), res.status);
+        }
+      } catch (err) {
+        console.error("NetWork Error", err);
+        options?.onError(err as Error);
+        options?.onMessage(
+          "Image generation has been cancelled.",
+          null,
+          IMAGE_ERROR,
+          true,
+        );
+      }
+    }
+    fetchImageAndUpdateMessage();
+  }
+}
 
 export async function requestChatStream(
   messages: Message[],