Handle uploads

packages/chat-core/src/index.ts

@@ -127,6 +127,10 @@ export interface BotResponseMetadata {
    * Whether the current conversation has been marked as incomprehension.
    */
   incomprehension?: boolean;
+  /**
+   * Upload URL's
+   */
+  uploadUrls: UploadUrl[];
   /**
    * Whether the client should poll for more bot responses.
    */
@@ -176,6 +180,20 @@ export interface BotMessage {
   selectedChoiceId?: string;
 }
 
+/**
+ * The upload destination for handling conversing with files
+ */
+export interface UploadUrl {
+  /**
+   * The URL of the upload
+   */
+  url: string;
+  /**
+   * The ID of the upload
+   */
+  uploadId: string;
+}
+
 /**
  * A choices to show to the user.
  */
@@ -401,6 +419,14 @@ export interface StructuredRequest {
    * The slots to populate
    */
   slots?: SlotsRecordOrArray;
+  /**
+   * Upload ID
+   */
+  uploadIds?: string[];
+  /**
+   * Upload utterance
+   */
+  utterance?: string;
   /**
    * @hidden
    * This is used internally to indicate that the client is polling the bot for more data.

packages/chat-widget/src/icons.tsx

@@ -24,14 +24,30 @@ export const ChatIcon = (): ReactNode => (
   </svg>
 );
 
+// eslint-disable-next-line jsdoc/require-returns
+/** @hidden @internal */
+export const CheckIcon = (): ReactNode => (
+  <svg viewBox="0 0 24 24" stroke="none" fill="currentColor">
+    <path d="M9 16.17 4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z" />
+  </svg>
+);
+
 // eslint-disable-next-line jsdoc/require-returns
 /** @hidden @internal */
 export const SendIcon = (): ReactNode => (
-  <svg viewBox="0 0 360 360" stroke="none" fill="currentColor">
+  <svg viewBox="0 0 24 24" stroke="none" fill="currentColor">
     <path d="M2.01 21 23 12 2.01 3 2 10l15 2-15 2z" />
   </svg>
 );
 
+// eslint-disable-next-line jsdoc/require-returns
+/** @hidden @internal */
+export const AddPhotoIcon = (): ReactNode => (
+  <svg viewBox="0 0 24 24" stroke="none" fill="currentColor">
+    <path d="M19 7v2.99s-1.99.01-2 0V7h-3s.01-1.99 0-2h3V2h2v3h3v2zm-3 4V8h-3V5H5c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h12c1.1 0 2-.9 2-2v-8zM5 19l3-4 2 3 3-4 4 5z" />
+  </svg>
+);
+
 // eslint-disable-next-line jsdoc/require-returns
 /** @hidden @internal */
 export const DownloadIcon = (): ReactNode => (

packages/chat-widget/src/index.tsx

@@ -2,6 +2,7 @@ import { marked, type MarkedExtension } from "marked";
 import {
   type FC,
   type ReactNode,
+  type ChangeEvent,
   createRef,
   useEffect,
   useCallback,
@@ -18,17 +19,21 @@ import { ThemeProvider } from "@emotion/react";
 import { useChat, type ChatHook } from "@nlxai/chat-react";
 import {
   type Response,
+  type BotResponse,
   type ConversationHandler,
+  type UploadUrl,
   getCurrentExpirationTimestamp,
 } from "@nlxai/chat-core";
 import {
   CloseIcon,
   MinimizeIcon,
   ChatIcon,
   SendIcon,
+  CheckIcon,
+  AddPhotoIcon,
   ErrorOutlineIcon,
 } from "./icons";
-import { last, equals } from "ramda";
+import { last, equals, findLast } from "ramda";
 import * as constants from "./ui/constants";
 import {
   type Props,
@@ -167,6 +172,7 @@ marked.use(markdownRendererOverrides);
 const MessageGroups: FC<{
   chat: ChatHook;
   children?: ReactNode;
+  uploadedFiles: Record<string, File>;
   customModalities: Record<string, CustomModalityComponent>;
   allowChoiceReselection?: boolean;
 }> = (props) => (
@@ -255,18 +261,54 @@ const MessageGroups: FC<{
         );
       }
 
-      if (response.type === "user" && response.payload.type === "text") {
-        return (
-          <C.MessageGroup key={responseIndex}>
-            <C.Message type="user">
-              <C.MessageBody
-                dangerouslySetInnerHTML={{
-                  __html: marked(response.payload.text),
-                }}
-              />
-            </C.Message>
-          </C.MessageGroup>
-        );
+      if (response.type === "user") {
+        if (response.payload.type === "text") {
+          return (
+            <C.MessageGroup key={responseIndex}>
+              <C.Message type="user">
+                <C.MessageBody
+                  dangerouslySetInnerHTML={{
+                    __html: marked(response.payload.text),
+                  }}
+                />
+              </C.Message>
+            </C.MessageGroup>
+          );
+        }
+
+        if (response.payload.type === "structured") {
+          const { uploadIds, utterance } = response.payload;
+          if (uploadIds == null) {
+            return null;
+          }
+          return (
+            <C.MessageGroup key={responseIndex}>
+              <C.Message type="user">
+                {utterance != null ? (
+                  <C.MessageBody
+                    dangerouslySetInnerHTML={{
+                      __html: marked(utterance),
+                    }}
+                  />
+                ) : null}
+              </C.Message>
+              {uploadIds.map((uploadId) => {
+                const file = props.uploadedFiles[uploadId];
+                if (file != null) {
+                  return (
+                    <img
+                      key={uploadId}
+                      style={{ maxWidth: "100%", borderRadius: "10px" }}
+                      src={URL.createObjectURL(file)}
+                      alt={file.name}
+                    />
+                  );
+                }
+                return null;
+              })}
+            </C.MessageGroup>
+          );
+        }
       }
 
       return null;
@@ -352,6 +394,77 @@ const isInputDisabled = (responses: Response[]): boolean => {
   return new URLSearchParams(payload).get("nlx:input-disabled") === "true";
 };
 
+const findActiveUpload = (responses: Response[]): UploadUrl | null => {
+  const lastBotResponse = findLast(
+    (response): response is BotResponse => response.type === "bot",
+    responses,
+  );
+  if (lastBotResponse == null) {
+    return null;
+  }
+  return lastBotResponse.payload.metadata?.uploadUrls?.[0] ?? null;
+};
+
+interface ImageUploadProps {
+  upload: UploadUrl;
+  onNewFile: (file: File) => void;
+}
+
+type UploadStatus = "empty" | "uploading" | "uploaded";
+
+const ImageUpload: FC<ImageUploadProps> = (props) => {
+  const [status, setStatus] = useState<UploadStatus>("empty");
+
+  if (status === "uploading") {
+    return <C.PendingMessageDots />;
+  }
+
+  if (status === "uploaded") {
+    return (
+      <C.UploadSuccess>
+        <CheckIcon />
+      </C.UploadSuccess>
+    );
+  }
+
+  const handleChange = async (
+    ev: ChangeEvent<HTMLInputElement>,
+  ): Promise<void> => {
+    const file = ev.target.files?.[0];
+    if (file == null) {
+      return;
+    }
+    setStatus("uploading");
+    try {
+      await fetch(props.upload.url, {
+        method: "PUT",
+        headers: {
+          "Content-Type": "image/jpeg",
+        },
+        body: file,
+      });
+      props.onNewFile(file);
+      setStatus("uploaded");
+    } catch (_err) {
+      // TODO: add error handling
+      setStatus("empty");
+    }
+  };
+
+  return (
+    <C.UploadIconLabel>
+      <input
+        type="file"
+        onChange={(ev) => {
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
+          handleChange(ev);
+        }}
+      />
+      <AddPhotoIcon />
+    </C.UploadIconLabel>
+  );
+};
+
 /**
  * Hook to get the ConversationHandler for the widget.
  * This may be called before the Widget has been created.
@@ -546,9 +659,26 @@ export const Widget = forwardRef<WidgetRef, Props>(function Widget(props, ref) {
     scrollToBottom();
   }, [chat.responses]);
 
+  const [uploadedFiles, setUploadedFiles] = useState<Record<string, File>>({});
+
+  /**
+   * IMPORTANT: upload state will get wiped out if there is a newer bot response comes in with different upload configuration.
+   * This is generally the way conversations are intended to work (choice buttons also do not work by default after new messages come in),
+   * and it's worth keeping these limitations in mind when designing/maintaining state management.
+   */
+  const activeUpload = findActiveUpload(chat.responses);
+
   const submit =
     chat.inputValue.replace(/ /gi, "") !== "" &&
     (() => {
+      if (activeUpload !== null) {
+        chat.conversationHandler.sendStructured({
+          uploadIds: [activeUpload.uploadId],
+          utterance: chat.inputValue,
+        });
+        chat.setInputValue("");
+        return;
+      }
       chat.conversationHandler.sendText(chat.inputValue);
       chat.setInputValue("");
     });
@@ -616,6 +746,7 @@ export const Widget = forwardRef<WidgetRef, Props>(function Widget(props, ref) {
                 ) : null}
                 <MessageGroups
                   chat={chat}
+                  uploadedFiles={uploadedFiles}
                   customModalities={props.customModalities ?? {}}
                   allowChoiceReselection={props.allowChoiceReselection}
                 >
@@ -652,6 +783,18 @@ export const Widget = forwardRef<WidgetRef, Props>(function Widget(props, ref) {
                       }}
                     />
                     <C.BottomButtonsContainer>
+                      {activeUpload == null ? null : (
+                        <ImageUpload
+                          key={activeUpload.uploadId}
+                          upload={activeUpload}
+                          onNewFile={(file) => {
+                            setUploadedFiles((prev) => ({
+                              ...prev,
+                              [activeUpload.uploadId]: file,
+                            }));
+                          }}
+                        />
+                      )}
                       <C.IconButton
                         disabled={Boolean(submit === false)}
                         onClick={() => {