feat(langgraph): handle MessageContentComplex types (#964)

Yonom · Oct 10, 2024 · ce93e73 · ce93e73
1 parent cf872da
commit ce93e73
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 2 deletions.
diff --git a/.changeset/silly-jeans-yell.md b/.changeset/silly-jeans-yell.md
@@ -0,0 +1,5 @@
+---
+"@assistant-ui/react-langgraph": patch
+---
+
+feat: handle MessageContentComplex types
diff --git a/packages/react-langgraph/src/convertLangchainMessages.ts b/packages/react-langgraph/src/convertLangchainMessages.ts
@@ -3,6 +3,7 @@
 import { useExternalMessageConverter } from "@assistant-ui/react";
 import { LangChainMessage } from "./types";
 import { ToolCallContentPart } from "@assistant-ui/react";
+import { ThreadUserMessage } from "@assistant-ui/react";
 
 export const convertLangchainMessages: useExternalMessageConverter.Callback<
   LangChainMessage
@@ -15,10 +16,42 @@ export const convertLangchainMessages: useExternalMessageConverter.Callback<
         content: [{ type: "text", text: message.content }],
       };
     case "human":
+      let content: ThreadUserMessage["content"];
+
+      if (typeof message.content === "string") {
+        content = [{ type: "text" as const, text: message.content }];
+      } else {
+        content = message.content.map((part) => {
+          if (typeof part === "string") {
+            return { type: "text", text: part };
+          } else {
+            const type = part.type;
+            switch (type) {
+              case "text":
+                return { type: "text" as const, text: part.text };
+
+              case "image_url":
+                return {
+                  type: "image" as const,
+                  image:
+                    typeof part.image_url === "string"
+                      ? part.image_url
+                      : part.image_url.url,
+                };
+              default:
+                const unhandledType: never = type;
+                throw new Error(
+                  `Unhandled content part type: ${unhandledType}`,
+                );
+            }
+          }
+        });
+      }
+
       return {
         role: "user",
         id: message.id,
-        content: [{ type: "text", text: message.content }],
+        content,
       };
     case "ai":
       return {

diff --git a/packages/react-langgraph/src/types.ts b/packages/react-langgraph/src/types.ts
@@ -10,12 +10,31 @@ export type LangChainToolCall = {
   args: Record<string, unknown>;
 };
 
+type MessageContentText = {
+  type: "text";
+  text: string;
+};
+
+type MessageContentImageUrl = {
+  type: "image_url";
+  image_url: string | { url: string };
+};
+
+type MessageContentComplex = MessageContentText | MessageContentImageUrl;
+
+type MessageContent = string | MessageContentComplex[];
+
 export type LangChainMessage =
   | {
       id?: string;
-      type: "human" | "system";
+      type: "system";
       content: string;
     }
+  | {
+      id?: string;
+      type: "human";
+      content: MessageContent;
+    }
   | {
       id?: string;
       type: "tool";