Working chat with vision

contracts/contracts/ChatOracle.sol

@@ -165,6 +165,13 @@ contract ChatOracle is IOracle {
         return IChatGpt(callbackAddresses[promptId]).getMessageHistoryRoles(promptCallBackId);
     }
 
+   function getMessagesAndRoles(
+        uint promptId,
+        uint promptCallBackId
+    ) public view returns (IOracle.Message[] memory) {
+        return IChatGpt(callbackAddresses[promptId]).getMessageHistory(promptCallBackId);
+    }
+
     function createFunctionCall(
         uint functionCallbackId,
         string memory functionType,

contracts/contracts/OpenAiChatGptVision.sol

@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.9;
+
+// Uncomment this line to use console.log
+// import "hardhat/console.sol";
+import "./interfaces/IOracle.sol";
+
+contract OpenAiChatGptVision {
+
+    struct ChatRun {
+        address owner;
+        IOracle.Message[] messages;
+        uint messagesCount;
+    }
+
+    mapping(uint => ChatRun) public chatRuns;
+    uint private chatRunsCount;
+
+    event ChatCreated(address indexed owner, uint indexed chatId);
+
+    address private owner;
+    address public oracleAddress;
+
+    event OracleAddressUpdated(address indexed newOracleAddress);
+
+    IOracle.OpenAiRequest private config;
+
+    constructor(address initialOracleAddress) {
+        owner = msg.sender;
+        oracleAddress = initialOracleAddress;
+        chatRunsCount = 0;
+
+        config = IOracle.OpenAiRequest({
+            model : "gpt-4-turbo",
+            frequencyPenalty : 21, // > 20 for null
+            logitBias : "", // empty str for null
+            maxTokens : 1000, // 0 for null
+            presencePenalty : 21, // > 20 for null
+            responseFormat : "{\"type\":\"text\"}",
+            seed : 0, // null
+            stop : "", // null
+            temperature : 10, // Example temperature (scaled up, 10 means 1.0), > 20 means null
+            topP : 101, // Percentage 0-100, > 100 means null
+            tools : "",
+            toolChoice : "", // "none" or "auto"
+            user : "" // null
+        });
+    }
+
+    modifier onlyOwner() {
+        require(msg.sender == owner, "Caller is not owner");
+        _;
+    }
+
+    modifier onlyOracle() {
+        require(msg.sender == oracleAddress, "Caller is not oracle");
+        _;
+    }
+
+    function setOracleAddress(address newOracleAddress) public onlyOwner {
+        oracleAddress = newOracleAddress;
+        emit OracleAddressUpdated(newOracleAddress);
+    }
+
+    function startChat(string memory message, string memory imageUrl) public returns (uint i) {
+        ChatRun storage run = chatRuns[chatRunsCount];
+
+        run.owner = msg.sender;
+        IOracle.Message memory newMessage = IOracle.Message({
+            role: "user",
+            content: new IOracle.Content[](2)
+        });
+        newMessage.content[0] = IOracle.Content({
+            contentType: "text",
+            value: message
+        });
+        newMessage.content[1] = IOracle.Content({
+            contentType: "image_url",
+            value: imageUrl
+        });
+        run.messages.push(newMessage);
+        run.messagesCount = 1;
+
+        uint currentId = chatRunsCount;
+        chatRunsCount = chatRunsCount + 1;
+
+        IOracle(oracleAddress).createOpenAiLlmCall(currentId, config);
+        emit ChatCreated(msg.sender, currentId);
+
+        return currentId;
+    }
+
+    function onOracleOpenAiLlmResponse(
+        uint runId,
+        IOracle.OpenAiResponse memory response,
+        string memory errorMessage
+    ) public onlyOracle {
+        ChatRun storage run = chatRuns[runId];
+        require(
+            keccak256(abi.encodePacked(run.messages[run.messagesCount - 1].role)) == keccak256(abi.encodePacked("user")),
+            "No message to respond to"
+        );
+
+        if (!compareStrings(errorMessage, "")) {
+            IOracle.Message memory newMessage = IOracle.Message({
+                role: "assistant",
+                content: new IOracle.Content[](1)
+            });
+            newMessage.content[0].contentType = "text";
+            newMessage.content[0].value = errorMessage;
+            run.messages.push(newMessage);
+            run.messagesCount++;
+        } else {
+            if (compareStrings(response.content, "")) {
+                IOracle(oracleAddress).createFunctionCall(runId, response.functionName, response.functionArguments);
+            } else {
+                IOracle.Message memory newMessage = IOracle.Message({
+                    role: "assistant",
+                    content: new IOracle.Content[](1)
+                });
+                newMessage.content[0].contentType = "text";
+                newMessage.content[0].value = response.content;
+                run.messages.push(newMessage);
+                run.messagesCount++;
+            }
+        }
+    }
+
+    function onOracleFunctionResponse(
+        uint runId,
+        string memory response,
+        string memory errorMessage
+    ) public onlyOracle {
+        ChatRun storage run = chatRuns[runId];
+        require(
+            compareStrings(run.messages[run.messagesCount - 1].role, "user"),
+            "No function to respond to"
+        );
+        if (compareStrings(errorMessage, "")) {
+             IOracle.Message memory newMessage = IOracle.Message({
+                role: "assistant",
+                content: new IOracle.Content[](1)
+            });
+            newMessage.content[0].contentType = "text";
+            newMessage.content[0].value = response;
+            run.messages.push(newMessage);
+            run.messagesCount++;
+            IOracle(oracleAddress).createOpenAiLlmCall(runId, config);
+        }
+    }
+
+    function addMessage(string memory message, uint runId) public {
+        ChatRun storage run = chatRuns[runId];
+        require(
+            keccak256(abi.encodePacked(run.messages[run.messagesCount - 1].role)) == keccak256(abi.encodePacked("assistant")),
+            "No response to previous message"
+        );
+        require(
+            run.owner == msg.sender, "Only chat owner can add messages"
+        );
+
+         IOracle.Message memory newMessage = IOracle.Message({
+            role: "user",
+            content: new IOracle.Content[](1)
+        });
+        newMessage.content[0].contentType = "text";
+        newMessage.content[0].value = message;
+        run.messages.push(newMessage);
+        run.messagesCount++;
+
+        IOracle(oracleAddress).createOpenAiLlmCall(runId, config);
+    }
+
+    function getMessageHistory(uint chatId) public view returns (IOracle.Message[] memory) {
+        return chatRuns[chatId].messages;
+    }
+
+    function compareStrings(string memory a, string memory b) private pure returns (bool) {
+        return (keccak256(abi.encodePacked((a))) == keccak256(abi.encodePacked((b))));
+    }
+}

contracts/contracts/interfaces/IChatGpt.sol

@@ -22,6 +22,10 @@ interface IChatGpt {
         string memory errorMessage
     ) external;
 
+    function getMessageHistory(
+        uint callbackId
+    ) external view returns (IOracle.Message[] memory);
+
     function getMessageHistoryContents(
         uint callbackId
     ) external view returns (string[] memory);

contracts/contracts/interfaces/IOracle.sol

@@ -2,8 +2,19 @@
 pragma solidity ^0.8.13;
 
 interface IOracle {
+
+    struct Content {
+        string contentType;
+        string value;
+    }
+
+    struct Message {
+        string role;
+        Content [] content;
+    }
+
     struct OpenAiRequest {
-        // "gpt-4-turbo-preview" or "gpt-3.5-turbo-1106"
+        // "gpt-4-turbo", "gpt-4-turbo-preview" or "gpt-3.5-turbo-1106"
         string model;
         // int -20 - 20, Mapped to float -2.0 - 2.0. If bigger than 20 then null
         int8 frequencyPenalty;

contracts/scripts/deployAll.ts

@@ -12,7 +12,7 @@ async function main() {
   await deployAgent(oracleAddress);
   console.log()
   await deployChatGptWithKnowledgeBase("ChatGpt", oracleAddress, "");
-  for (let contractName of ["OpenAiChatGpt", "GroqChatGpt"]) {
+  for (let contractName of ["OpenAiChatGpt", "GroqChatGpt", "OpenAiChatGptVision"]) {
     await deployChatGpt(contractName, oracleAddress)
   }
 }