E2E tests (#56)

galadriel-ai · Apr 8, 2024 · 5522f24 · 5522f24
1 parent f05f73c
commit 5522f24
Show file tree

Hide file tree

Showing 11 changed files with 809 additions and 59 deletions.
diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml
@@ -0,0 +1,125 @@
+
+name: e2e tests
+on:
+  pull_request:
+    branches:
+      - main
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    env:
+      NETWORK: "localhost"
+      ORACLE_ADDRESS: "0x5FbDB2315678afecb367f032d93F642f64180aa3"
+      TEST_CONTRACT_ADDRESS: "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318"
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: contracts/package-lock.json
+    - name: "Install Dependencies"
+      id: install
+      run: cd contracts && npm ci
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Cache pip
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pip
+          !~/.cache/pip/log
+        key: ${{ runner.os }}-pip-${{ hashFiles('oracles/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install Python Dependencies
+      run: pip install -r oracles/requirements.txt
+    - name: Run Hardhat Node
+      run: cd contracts && npx hardhat node &
+    - name: "Deploy Contracts"
+      run:
+        cd contracts && npm run deployAll:localhost
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Whitelist the address"
+      run:
+        cd contracts && npx hardhat whitelist --oracle-address ${{ env.ORACLE_ADDRESS }} --whitelist-address 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 --network localhost
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Deploy Test Contract"
+      run:
+        cd contracts && npm run deployTest:localhost
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+        ORACLE_ADDRESS: ${{ env.ORACLE_ADDRESS }}
+    - name: Run Python Oracle
+      run: | 
+        cd oracles && python oracle.py > oracle_output.txt 2>&1 &
+        echo $! > python_oracle.pid
+      env:
+        CHAIN_ID: 1337
+        WEB3_RPC_URL: "http://127.0.0.1:8545/"
+        ORACLE_ADDRESS: ${{ env.ORACLE_ADDRESS }}
+        PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
+        OPEN_AI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+        SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
+        BEARLY_API_KEY: ${{ secrets.BEARLY_API_KEY }}
+        PINATA_GATEWAY_TOKEN: ${{ secrets.PINATA_GATEWAY_TOKEN }}
+        PINATA_API_JWT: ${{ secrets.PINATA_API_JWT }}
+    - name: "Test 1 - OpenAI gpt-4-turbo-preview"
+      run:
+        cd contracts && npx hardhat openai --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --model gpt-4-turbo-preview --message "Who is the president of USA?" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 2 - OpenAI gpt-3.5-turbo-1106"
+      run:
+        cd contracts && npx hardhat openai --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --model gpt-3.5-turbo-1106 --message "Who is the president of USA?" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 3 - Groq llama2-70b-4096"
+      run:
+        cd contracts && npx hardhat groq --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --model llama2-70b-4096 --message "Who is the president of USA?" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 4 - Groq mixtral-8x7b-32768"
+      run:
+        cd contracts && npx hardhat groq --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --model mixtral-8x7b-32768 --message "Who is the president of USA?" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 5 - gemma-7b-it"
+      run:
+        cd contracts && npx hardhat groq --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --model gemma-7b-it --message "Who is the president of USA?" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 6 - Image Generation"
+      run:
+        cd contracts && npx hardhat image_generation --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --query "Red rose" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 7 - Web search"
+      run:
+        cd contracts && npx hardhat web_search --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --query "Capital of Germany" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 8 - Code interpreter"
+      run:
+        cd contracts && npx hardhat code_interpreter --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --query "print(2+2)" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 9 - Add knowledge base"
+      run:
+        cd contracts && npx hardhat add_knowledge_base --oracle-address ${{ env.ORACLE_ADDRESS }} --cid QmdCgbMawRVE6Kc1joZmhgDo2mSZFgRgWvBCqUvJV9JwkF --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: "Test 10 - Query knowledge base"
+      run:
+        cd contracts && npx hardhat query_knowledge_base --contract-address ${{ env.TEST_CONTRACT_ADDRESS }} --cid QmdCgbMawRVE6Kc1joZmhgDo2mSZFgRgWvBCqUvJV9JwkF --query "What is the oracle smart contract address?" --network ${{ env.NETWORK }}
+      env:
+        PRIVATE_KEY_LOCALHOST: ${{ secrets.PRIVATE_KEY }}
+    - name: Display Oracle Output
+      if: always() # Ensures this step runs even if a previous step fails
+      run: cat oracles/oracle_output.txt
diff --git a/contracts/README.md b/contracts/README.md
@@ -77,3 +77,27 @@ npm run generateStandardJson
 
 This generated JSON files are in `./contracts/artifacts/solidity-json/contracts`
 
+### Running e2e validation tests
+
+**Deploy test contract to relevant network**
+```
+npm run deployTest:localhost
+```
+```
+npm run deployTest:galadriel
+```
+
+**Single run**
+```
+npx hardhat e2e --contract-address <Test contract address> --oracle-address <oracle contract address> --network <network>
+```
+
+**Cron job with Slack**
+```
+ts-node tasks/e2eCron.ts
+```
+
+**Cron job with Slack in docker**
+```
+docker compose -f docker/docker-compose-e2e.yml up --build -d
+```
diff --git a/contracts/contracts/Test.sol b/contracts/contracts/Test.sol
@@ -6,6 +6,78 @@ pragma solidity ^0.8.20;
 
 
 interface IOracle {
+    struct GroqRequest {
+        string model;
+        int8 frequencyPenalty;
+        string logitBias;
+        uint32 maxTokens;
+        int8 presencePenalty;
+        string responseFormat;
+        uint seed;
+        string stop;
+        uint temperature;
+        uint topP;
+        string user;
+    }
+
+    struct GroqResponse {
+        string id;
+
+        string content;
+
+        uint64 created;
+        string model;
+        string systemFingerprint;
+        string object;
+
+        uint32 completionTokens;
+        uint32 promptTokens;
+        uint32 totalTokens;
+    }
+
+    function createGroqLlmCall(
+        uint promptId,
+        GroqRequest memory request
+    ) external returns (uint);
+
+    struct OpenAiRequest {
+        string model;
+        int8 frequencyPenalty;
+        string logitBias;
+        uint32 maxTokens;
+        int8 presencePenalty;
+        string responseFormat;
+        uint seed;
+        string stop;
+        uint temperature;
+        uint topP;
+        string tools;
+        string toolChoice;
+        string user;
+    }
+
+    struct OpenAiResponse {
+        string id;
+
+        string content;
+        string functionName;
+        string functionArguments;
+
+        uint64 created;
+        string model;
+        string systemFingerprint;
+        string object;
+
+        uint32 completionTokens;
+        uint32 promptTokens;
+        uint32 totalTokens;
+    }
+
+    function createOpenAiLlmCall(
+        uint promptId,
+        OpenAiRequest memory request
+    ) external returns (uint);
+
     function createFunctionCall(
         uint functionCallbackId,
         string memory functionType,
@@ -23,6 +95,7 @@ interface IOracle {
 contract Test {
     address private owner;
     address public oracleAddress;
+    string public llmMessage;
     string public lastResponse;
     string public lastError;
     uint private callsCount;
@@ -67,6 +140,76 @@ contract Test {
         return currentId;
     }
 
+    function callOpenAiLLM(string memory model, string memory message) public returns (uint i) {
+        uint currentId = callsCount;
+        callsCount = currentId + 1;
+
+        llmMessage = message;
+        lastResponse = "";
+        lastError = "";
+
+        IOracle(oracleAddress).createOpenAiLlmCall(
+            currentId,
+            IOracle.OpenAiRequest({
+                model: model,
+                frequencyPenalty : 21, // > 20 for null
+                logitBias : "", // empty str for null
+                maxTokens : 1000, // 0 for null
+                presencePenalty : 21, // > 20 for null
+                responseFormat : "{\"type\":\"text\"}",
+                seed : 0, // null
+                stop : "", // null
+                temperature : 10, // Example temperature (scaled up, 10 means 1.0), > 20 means null
+                topP : 101, // Percentage 0-100, > 100 means null
+                tools : "",
+                toolChoice : "", // "none" or "auto"
+                user : "" // null
+            })
+        );
+
+        return currentId;
+    }
+
+    function callGroqLLM(string memory model, string memory message) public returns (uint i) {
+        uint currentId = callsCount;
+        callsCount = currentId + 1;
+
+        llmMessage = message;
+        lastResponse = "";
+        lastError = "";
+
+        IOracle(oracleAddress).createGroqLlmCall(
+            currentId,
+            IOracle.GroqRequest({
+                model : model,
+                frequencyPenalty : 21, // > 20 for null
+                logitBias : "", // empty str for null
+                maxTokens : 1000, // 0 for null
+                presencePenalty : 21, // > 20 for null
+                responseFormat : "{\"type\":\"text\"}",
+                seed : 0, // null
+                stop : "", // null
+                temperature : 10, // Example temperature (scaled up, 10 means 1.0), > 20 means null
+                topP : 101, // Percentage 0-100, > 100 means null
+                user : "" // null
+            })
+        );
+
+        return currentId;
+    }
+
+    function getMessageHistoryContents(uint chatId) public view returns (string[] memory) {
+        string[] memory messages = new string[](1);
+        messages[0] = llmMessage;
+        return messages;
+    }
+
+    function getMessageHistoryRoles(uint chatId) public view returns (string[] memory) {
+        string[] memory roles = new string[](1);
+        roles[0] = "user";
+        return roles;
+    }
+
     function queryKnowledgeBase(string memory cid, string memory query) public returns (uint i) {
         uint currentId = callsCount;
         callsCount = currentId + 1;
@@ -97,11 +240,29 @@ contract Test {
             string [] memory documents,
             string memory errorMessage
         ) public onlyOracle {
-            string memory newContent = "";
-            for (uint i = 0; i < documents.length; i++) {
-                newContent = string(abi.encodePacked(newContent, documents[i], "\n"));
-            }
-            lastResponse = newContent;
-            lastError = errorMessage;
+        string memory newContent = "";
+        for (uint i = 0; i < documents.length; i++) {
+            newContent = string(abi.encodePacked(newContent, documents[i], "\n"));
         }
+        lastResponse = newContent;
+        lastError = errorMessage;
+    }
+
+    function onOracleOpenAiLlmResponse(
+        uint runId,
+        IOracle.OpenAiResponse memory response,
+        string memory errorMessage
+    ) public onlyOracle {
+        lastResponse = response.content;
+        lastError = errorMessage;
     }
+
+    function onOracleGroqLlmResponse(
+        uint runId,
+        IOracle.GroqResponse memory response,
+        string memory errorMessage
+    ) public onlyOracle {
+        lastResponse = response.content;
+        lastError = errorMessage;
+    }
+}
diff --git a/contracts/docker/Dockerfile b/contracts/docker/Dockerfile
@@ -0,0 +1,11 @@
+FROM node:20-alpine
+
+# Set the working directory in the container
+WORKDIR /app
+
+COPY . .
+
+# Install any dependencies
+RUN npm ci
+
+CMD ["npx", "ts-node", "tasks/e2eCron.ts"]
diff --git a/contracts/docker/docker-compose-e2e.yml b/contracts/docker/docker-compose-e2e.yml
@@ -0,0 +1,9 @@
+version: '3'
+services:
+  app:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    volumes:
+      - ..:/app
+    restart: "unless-stopped"
diff --git a/contracts/hardhat.config.ts b/contracts/hardhat.config.ts
@@ -3,6 +3,7 @@ import "@nomicfoundation/hardhat-toolbox";
 import "@xyrusworx/hardhat-solidity-json";
 import "./tasks/whitelist";
 import "./tasks/deployments";
+import "./tasks/e2e";
 import "./tasks/functions";
 
 require('dotenv').config()