e2e cron job improvement (#73)

contracts/tasks/e2e.ts

@@ -19,69 +19,72 @@ task("e2e", "Runs all e2e tests")
     const contractAddress = taskArgs.contractAddress;
     const oracleAddress = taskArgs.oracleAddress;
 
-    await runOpenAi(
+    process.env.RUN_MODE = "e2e-script";
+
+    const testResults: Record<string, string> = {};
+
+    let result = await runOpenAi(
       contractAddress,
       "gpt-4-turbo-preview",
       "Who is the president of USA?",
       hre,
     )
-    await runOpenAi(
+    testResults["OpenAI gpt-4-turbo-preview"] = result.error || "✅";
+    result = await runOpenAi(
       contractAddress,
       "gpt-3.5-turbo-1106",
       "Who is the president of USA?",
       hre,
     )
-    await runGroq(
+    testResults["OpenAI gpt-3.5-turbo-1106"] = result.error || "✅";
+    result = await runGroq(
       contractAddress,
       "llama2-70b-4096",
       "Who is the president of USA?",
       hre,
     )
-    await runGroq(
+    testResults["Groq llama2-70b-4096"] = result.error || "✅";
+    result = await runGroq(
       contractAddress,
       "mixtral-8x7b-32768",
       "Who is the president of USA?",
       hre,
     )
-    await runGroq(
+    testResults["Groq mixtral-8x7b-32768"] = result.error || "✅";
+    result = await runGroq(
       contractAddress,
       "gemma-7b-it",
       "Who is the president of USA?",
       hre,
     )
-
-    console.log(`Running "image_generation"`)
-    await runTaskWithTimeout(
+    testResults["gemma-7b-it mixtral-8x7b-32768"] = result.error || "✅";
+    result = await runTaskWithTimeout(
       "image_generation",
       {
         contractAddress,
         query: "Red rose",
       },
       hre,
     )
-    console.log(`DONE Running "image_generation"`)
-
-    console.log(`Running "web_search"`)
-    await runTaskWithTimeout(
+    testResults["OpenAI image_generation"] = result.error || "✅";
+    result = await runTaskWithTimeout(
       "web_search",
       {
         contractAddress,
         query: "Capital of Germany",
       },
       hre,
     )
-    console.log(`DONE Running "web_search"`)
-
-    console.log(`Running "code_interpreter"`)
-    await runTaskWithTimeout(
+    testResults["web_search"] = result.error || "✅";
+    result = await runTaskWithTimeout(
       "code_interpreter",
       {
         contractAddress,
         query: "print(2+2)",
       },
       hre,
     )
-    console.log(`DONE Running "code_interpreter"`)
+    testResults["code_interpreter"] = result.error || "✅";
 
     // console.log(`Running "add_knowledge_base"`)
     // await runTaskWithTimeout(
@@ -93,8 +96,7 @@ task("e2e", "Runs all e2e tests")
     //   hre,
     // )
     // console.log(`DONE Running "add_knowledge_base"`)
-    console.log(`Running "query_knowledge_base"`)
-    await runTaskWithTimeout(
+    result = await runTaskWithTimeout(
       "query_knowledge_base",
       {
         contractAddress,
@@ -103,18 +105,27 @@ task("e2e", "Runs all e2e tests")
       },
       hre,
     )
-    console.log(`DONE Running "query_knowledge_base"`)
-
-
-    console.log("================================================")
-    console.log(green, "e2e run done", reset)
+    testResults["query_knowledge_base"] = result.error || "✅";
+    const totalTests = Object.keys(testResults).length;
+    const passedTests = Object.values(testResults).filter(result => result === "✅").length;
+    const failedTests = totalTests - passedTests;
+    console.log(`${passedTests} out of ${totalTests} tests passed `);
+    const transformedResults = Object.entries(testResults).map(([testName, result]) => ({
+      "Test": testName,
+      "Result": result
+    }));
+
+    console.table(transformedResults);
+    if (failedTests > 0) {
+      process.exit(1);
+    }
   });
 
 async function runTaskWithTimeout(
   taskIdentifier: string,
   taskArguments: any,
   hre: HardhatRuntimeEnvironment,
-) {
+): Promise<any> {
   try {
     const timeoutPromise = new Promise((resolve, reject) => {
       const id = setTimeout(() => {
@@ -123,15 +134,15 @@ async function runTaskWithTimeout(
       }, TIMEOUT_SECONDS * 1000);
     });
 
-    await Promise.race([
+    const taskResult = await Promise.race([
       timeoutPromise,
       hre.run(taskIdentifier, taskArguments),
     ]);
+    return taskResult;
   } catch (e: any) {
     process.stderr.write(e.message + " ")
     throw e
   }
-
 }
 
 
@@ -141,8 +152,7 @@ async function runOpenAi(
   message: string,
   hre: HardhatRuntimeEnvironment,
 ) {
-  console.log(`Running "openai", with model: ${model}`)
-  await runTaskWithTimeout(
+  let result = await runTaskWithTimeout(
     "openai",
     {
       contractAddress,
@@ -151,7 +161,7 @@ async function runOpenAi(
     },
     hre,
   )
-  console.log(`DONE Running "openai", with model: ${model}.`)
+  return result;
 }
 
 async function runGroq(
@@ -160,8 +170,7 @@ async function runGroq(
   message: string,
   hre: HardhatRuntimeEnvironment,
 ) {
-  console.log(`Running "groq", with model: ${model}`)
-  await runTaskWithTimeout(
+  let result = await runTaskWithTimeout(
     "groq",
     {
       contractAddress,
@@ -170,5 +179,5 @@ async function runGroq(
     },
     hre,
   )
-  console.log(`DONE Running "groq", with model: ${model}.`)
-}
+  return result;
+}

contracts/tasks/e2eCron.ts

@@ -34,9 +34,10 @@ async function main(): Promise<void> {
     "--network", network
   ];
 
+  let previousResult = false;
   while (true) {
     console.log(`Running e2e tests with `)
-    await runTests(command, args, slackWebHookUrl)
+    previousResult = await runTests(command, args, slackWebHookUrl, previousResult)
     console.log(`Run done, sleeping for ${TIMEOUT_SECONDS} seconds`)
     await new Promise((resolve) => setTimeout(resolve, TIMEOUT_SECONDS * 1000));
   }
@@ -46,44 +47,52 @@ async function runTests(
   command: string,
   args: string[],
   slackWebHookUrl: string,
-): Promise<void> {
+  previousResult: boolean
+): Promise<boolean> {
+  let isSuccess = false;
+  let stdoutData = '';
   try {
-    await new Promise((resolve, reject) => {
-      const childProcess = spawn(command, args, {shell: true});
-
-      let stdoutData = '';
-      let stderrData = '';
+    isSuccess = await new Promise((resolve) => {
+      const childProcess = spawn(command, args, { shell: true });
 
       childProcess.stdout.on('data', (data) => {
-        console.log(data.toString())
+        stdoutData += data.toString();
+        console.log(data.toString());
       });
 
       childProcess.stderr.on('data', (data) => {
-        stderrData += data.toString();
+        console.error(data.toString());
       });
 
       childProcess.on('close', (code) => {
         if (code === 0) {
-          resolve(stdoutData);
+          resolve(true);
         } else {
-          reject(new Error(stderrData));
+          resolve(false);
         }
       });
-    })
+    });
   } catch (e: any) {
-    console.error(e.message)
+    console.error(e.message);
     await postSlackMessage(
-      `e2e blockchain tests failed\n` +
-      "```" +
-      `${e.message}` +
-      "```",
+      e.message,
       slackWebHookUrl,
-    )
+    );
   }
+
+  // If isSuccess is false, you might want to send a message to Slack here as well
+  if (!isSuccess || !previousResult) {
+    await postSlackMessage(
+      stdoutData,
+      slackWebHookUrl,
+    );
+  }
+
+  return isSuccess;
 }
 
 async function postSlackMessage(
-  text: string,
+  output: string,
   webhookUrl: string,
 ) {
   try {
@@ -95,7 +104,15 @@ async function postSlackMessage(
           "Content-type": "application/json"
         },
         body: JSON.stringify({
-          text
+          blocks: [
+            {
+              type: "section",
+              text: {
+                type: "mrkdwn",
+                text: "```" + output +"```"
+              }
+            }
+          ]
         })
       }
     )