fix(langgraph): allow complex content in ai messages (#983)

Yonom · Oct 13, 2024 · ea90b84 · ea90b84
1 parent 0a3bd06
commit ea90b84
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 41 deletions.
diff --git a/.changeset/curvy-keys-joke.md b/.changeset/curvy-keys-joke.md
@@ -0,0 +1,5 @@
+---
+"@assistant-ui/react-langgraph": patch
+---
+
+fix(langgraph): allow complex content in ai messages
diff --git a/packages/react-langgraph/src/convertLangchainMessages.ts b/packages/react-langgraph/src/convertLangchainMessages.ts
@@ -5,6 +5,29 @@ import { LangChainMessage } from "./types";
 import { ToolCallContentPart } from "@assistant-ui/react";
 import { ThreadUserMessage } from "@assistant-ui/react";
 
+const contentToParts = (content: LangChainMessage["content"]) => {
+  if (typeof content === "string")
+    return [{ type: "text" as const, text: content }];
+  return content.map((part): ThreadUserMessage["content"][number] => {
+    switch (part.type) {
+      case "text":
+        return { type: "text", text: part.text };
+      case "image_url":
+        if (typeof part.image_url === "string") {
+          return { type: "image", image: part.image_url };
+        } else {
+          return {
+            type: "image",
+            image: part.image_url.url,
+          };
+        }
+      default:
+        const _exhaustiveCheck: never = part;
+        throw new Error(`Unknown content part type: ${_exhaustiveCheck}`);
+    }
+  });
+};
+
 export const convertLangchainMessages: useExternalMessageConverter.Callback<
   LangChainMessage
 > = (message) => {
@@ -16,52 +39,17 @@ export const convertLangchainMessages: useExternalMessageConverter.Callback<
         content: [{ type: "text", text: message.content }],
       };
     case "human":
-      let content: ThreadUserMessage["content"];
-
-      if (typeof message.content === "string") {
-        content = [{ type: "text" as const, text: message.content }];
-      } else {
-        content = message.content.map((part) => {
-          if (typeof part === "string") {
-            return { type: "text", text: part };
-          } else {
-            const type = part.type;
-            switch (type) {
-              case "text":
-                return { type: "text" as const, text: part.text };
-
-              case "image_url":
-                return {
-                  type: "image" as const,
-                  image:
-                    typeof part.image_url === "string"
-                      ? part.image_url
-                      : part.image_url.url,
-                };
-              default:
-                const unhandledType: never = type;
-                throw new Error(
-                  `Unhandled content part type: ${unhandledType}`,
-                );
-            }
-          }
-        });
-      }
-
       return {
         role: "user",
         id: message.id,
-        content,
+        content: contentToParts(message.content),
       };
     case "ai":
       return {
         role: "assistant",
         id: message.id,
         content: [
-          {
-            type: "text",
-            text: message.content,
-          },
+          ...contentToParts(message.content),
           ...(message.tool_calls?.map(
             (chunk): ToolCallContentPart => ({
               type: "tool-call",

diff --git a/packages/react-langgraph/src/types.ts b/packages/react-langgraph/src/types.ts
@@ -20,9 +20,11 @@ type MessageContentImageUrl = {
   image_url: string | { url: string };
 };
 
-type MessageContentComplex = MessageContentText | MessageContentImageUrl;
+type UserMessageContentComplex = MessageContentText | MessageContentImageUrl;
+type AssistantMessageContentComplex = MessageContentText;
 
-type MessageContent = string | MessageContentComplex[];
+type UserMessageContent = string | UserMessageContentComplex[];
+type AssistantMessageContent = string | AssistantMessageContentComplex[];
 
 export type LangChainMessage =
   | {
@@ -33,7 +35,7 @@ export type LangChainMessage =
   | {
       id?: string;
       type: "human";
-      content: MessageContent;
+      content: UserMessageContent;
     }
   | {
       id?: string;
@@ -45,7 +47,7 @@ export type LangChainMessage =
   | {
       id?: string;
       type: "ai";
-      content: string;
+      content: AssistantMessageContent;
       tool_call_chunks?: LangChainToolCallChunk[];
       tool_calls?: LangChainToolCall[];
     };