From 1ae8641bc392674fbd81b23456c0be72144a7aef Mon Sep 17 00:00:00 2001
From: fatwang2 <wangding0798@gmail.com>
Date: Mon, 3 Jun 2024 23:26:43 +0800
Subject: [PATCH] support siri ultra vision

---
 README.md      | 21 ++++++++++++++-------
 src/chat.ts    | 45 ++++++++++++++++++++++++++++++++++++---------
 src/history.ts |  2 --
 src/index.ts   |  2 --
 4 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index c474189..8de693a 100644
--- a/README.md
+++ b/README.md
@@ -4,21 +4,28 @@ This is a Siri Ultra that works with Apple Shortcuts removing the need for a ded
 
 ## How it works
 
-The assistant is run on Cloudflare Workers and can work with any LLM model. 
+The assistant is run on Cloudflare Workers and can work with any LLMs. 
 
-[Demo Video](https://x.com/fatwang2ai/status/1789601031313035281)
+[Siri Ultra Demo Video](https://x.com/fatwang2ai/status/1789601031313035281)
+[Siri Ultra Vision Demo Video](https://x.com/fatwang2ai/status/1791648375693361161)
+
+## Features
+- Real-time dialogue 💬
+- Real-time voice 🎙️
+- Real-time Web Search 🔍
+- Talk with pictures 🌄
+- Talk with videos 📹
 
 # Usage
 
 ## Method 1: Setting Up the Shortcut Directly
 1. **Install the Shortcut**: 
-   - Click [this link](https://search2ai.online/siri002) to install.
+   - Click [Siri Ultra](https://search2ai.online/siri002) for chat with LLMs to install.
+   - Click [Siri Ultra Vision](https://search2ai.online/siri003) for vision of LLMs to install.
 
 2. **Configure**: 
    - Open the Shortcut, follow prompts to input necessary variables.
 
-  > **Branch**: Siri Ultra Vision is testing on [this link](https://search2ai.online/siri003), which can talk with photos by gpt-4o
-
 ## Method 2: Setting Up the Self-Hosted Version
 
 ### Getting Started
@@ -66,8 +73,8 @@ To deploy the worker, run `npx wrangler deploy`.
 ### Setting Up the Shortcut
 
 1. **Install the shortcut**:
-   - Use [this link](https://search2ai.online/siri002) to install the shortcut.
-
+   - Use [Siri Ultra](https://search2ai.online/siri002) to install the chat shortcut.
+   - Use [Siri Ultra Vision](https://search2ai.online/siri003) to install the vision shortcut.
 2. **Configure the shortcut**:
    - Open the shortcut and replace the `URL` field with your worker's URL.
    - If you didn't change the default name, the URL should be `https://siri-ultra.<your-username>.workers.dev`.
diff --git a/src/chat.ts b/src/chat.ts
index 8f71695..b4e0da3 100644
--- a/src/chat.ts
+++ b/src/chat.ts
@@ -5,6 +5,7 @@ import { FunctionHandler } from "./functions";
 export interface IBody {
   chat_id: string;
   input: string;
+  image?: string[];
   date: string;
   config?: {
     model?: string;
@@ -55,18 +56,44 @@ export const handle = async (req: IRequest): Promise<string> => {
     date: ${req.request.date}
   `;
   const chat = ChatHistory.getInstance(req.env.siri_ai_chats);
-  await chat.add(req.request.chat_id, {
-    role: "user",
-    content: req.request.input,
-  });
+  if (!req.request.image) {
+    await chat.add(req.request.chat_id, {
+      role: "user",
+      content: req.request.input,
+    });
+  }
+
   let response = "";
   while (true) {
-    const ask = await openai.client.chat.completions.create({
-      model: openai.model,
-      messages: [
+    let currentMessages;
+    if (req.request.image) {
+      currentMessages = [
+        { role: "system", content: system },
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: req.request.input,
+            },
+            ...req.request.image.map((base64Image) => ({
+              type: "image_url",
+              image_url: {
+                url: `data:image/jpeg;base64,${base64Image}`,
+              },
+            })),
+          ],
+        },
+      ];
+    } else {
+      currentMessages = [
         { role: "system", content: system },
         ...(await chat.get(req.request.chat_id)),
-      ],
+      ];
+    }
+    const ask = await openai.client.chat.completions.create({
+      model: openai.model,
+      messages: currentMessages,
       tools: FunctionHandler.functions,
     });
     if (ask.choices[0].message.tool_calls) {
@@ -101,5 +128,5 @@ export const handle = async (req: IRequest): Promise<string> => {
       break;
     }
   }
-  return response;
+  return response;  
 };
\ No newline at end of file
diff --git a/src/history.ts b/src/history.ts
index fb2666d..79994fe 100644
--- a/src/history.ts
+++ b/src/history.ts
@@ -26,7 +26,6 @@ export class ChatHistory {
       messages.push(message);
       await this.kv.put(chat_id, JSON.stringify(messages));
     }
-    console.log(JSON.stringify(this.kv.get(chat_id), null, 2));
   }
 
   async get(chat_id: string): Promise<ChatEntry[]> {
@@ -34,7 +33,6 @@ export class ChatHistory {
     if (!chat) {
       return [];
     }
-    console.log(chat);
     return JSON.parse(chat);
   }
 }
diff --git a/src/index.ts b/src/index.ts
index a642157..a781e74 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -6,7 +6,6 @@ const app = new Hono();
 app.post("/", async (c) => {
   const body = (await c.req.json()) as IBody;
   try {
-    console.log(JSON.stringify(body, null, 2));
     const response = await handle({
       env: c.env,
       request: body,
@@ -16,7 +15,6 @@ app.post("/", async (c) => {
       response,
     });
   } catch (error) {
-    console.log(error);
     return c.json({
       response: "Something went wrong, we are working on it",
     });