Adds nice result printout

galadriel-ai · Apr 10, 2024 · 791091a · 791091a
1 parent cbfd970
commit 791091a
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 39 deletions.
diff --git a/contracts/tasks/e2e.ts b/contracts/tasks/e2e.ts
@@ -21,39 +21,43 @@ task("e2e", "Runs all e2e tests")
 
     process.env.RUN_MODE = "e2e-script";
 
+    const testResults: Record<string, string> = {};
+
     let result = await runOpenAi(
       contractAddress,
       "gpt-4-turbo-preview",
       "Who is the president of USA?",
       hre,
     )
-    console.log(result);
+    testResults["OpenAI gpt-4-turbo-preview"] = result.error.length ? result.error : "✅";
     result = await runOpenAi(
       contractAddress,
       "gpt-3.5-turbo-1106",
       "Who is the president of USA?",
       hre,
     )
+    testResults["OpenAI gpt-3.5-turbo-1106"] = result.error.length ? result.error : "✅";
     result = await runGroq(
       contractAddress,
       "llama2-70b-4096",
       "Who is the president of USA?",
       hre,
     )
+    testResults["Groq llama2-70b-4096"] = result.error.length ? result.error : "✅";
     result = await runGroq(
       contractAddress,
       "mixtral-8x7b-32768",
       "Who is the president of USA?",
       hre,
     )
+    testResults["Groq mixtral-8x7b-32768"] = result.error.length ? result.error : "✅";
     result = await runGroq(
       contractAddress,
       "gemma-7b-it",
       "Who is the president of USA?",
       hre,
     )
-
-    console.log(`Running "image_generation"`)
+    testResults["gemma-7b-it mixtral-8x7b-32768"] = result.error.length ? result.error : "✅";
     result = await runTaskWithTimeout(
       "image_generation",
       {
@@ -62,9 +66,7 @@ task("e2e", "Runs all e2e tests")
       },
       hre,
     )
-    console.log(`DONE Running "image_generation"`)
-
-    console.log(`Running "web_search"`)
+    testResults["OpenAI image_generation"] = result.error.length ? result.error : "✅";
     result = await runTaskWithTimeout(
       "web_search",
       {
@@ -73,9 +75,7 @@ task("e2e", "Runs all e2e tests")
       },
       hre,
     )
-    console.log(`DONE Running "web_search"`)
-
-    console.log(`Running "code_interpreter"`)
+    testResults["web_search"] = result.error.length ? result.error : "✅";
     result = await runTaskWithTimeout(
       "code_interpreter",
       {
@@ -84,7 +84,7 @@ task("e2e", "Runs all e2e tests")
       },
       hre,
     )
-    console.log(`DONE Running "code_interpreter"`)
+    testResults["code_interpreter"] = result.error.length ? result.error : "✅";
 
     // console.log(`Running "add_knowledge_base"`)
     // await runTaskWithTimeout(
@@ -96,7 +96,6 @@ task("e2e", "Runs all e2e tests")
     //   hre,
     // )
     // console.log(`DONE Running "add_knowledge_base"`)
-    console.log(`Running "query_knowledge_base"`)
     result = await runTaskWithTimeout(
       "query_knowledge_base",
       {
@@ -106,9 +105,20 @@ task("e2e", "Runs all e2e tests")
       },
       hre,
     )
-    console.log(`DONE Running "query_knowledge_base"`)
-    console.log("================================================")
-    console.log(green, "e2e run done", reset)
+    testResults["query_knowledge_base"] = result.error.length ? result.error : "✅";
+    const totalTests = Object.keys(testResults).length;
+    const passedTests = Object.values(testResults).filter(result => result === "✅").length;
+    const failedTests = totalTests - passedTests;
+    console.log(`${passedTests} out of ${totalTests} tests passed `);
+    const transformedResults = Object.entries(testResults).map(([testName, result]) => ({
+      "Test": testName,
+      "Result": result
+    }));
+
+    console.table(transformedResults);
+    if (failedTests > 0) {
+      process.exit(1);
+    }
   });
 
 async function runTaskWithTimeout(
@@ -142,7 +152,6 @@ async function runOpenAi(
   message: string,
   hre: HardhatRuntimeEnvironment,
 ) {
-  console.log(`Running "openai", with model: ${model}`)
   let result = await runTaskWithTimeout(
     "openai",
     {
@@ -152,7 +161,6 @@ async function runOpenAi(
     },
     hre,
   )
-  console.log(`DONE Running "openai", with model: ${model}.`)
   return result;
 }
 
@@ -162,7 +170,6 @@ async function runGroq(
   message: string,
   hre: HardhatRuntimeEnvironment,
 ) {
-  console.log(`Running "groq", with model: ${model}`)
   let result = await runTaskWithTimeout(
     "groq",
     {
@@ -172,6 +179,5 @@ async function runGroq(
     },
     hre,
   )
-  console.log(`DONE Running "groq", with model: ${model}.`)
   return result;
-}
+}
diff --git a/contracts/tasks/e2eCron.ts b/contracts/tasks/e2eCron.ts
@@ -35,8 +35,9 @@ async function main(): Promise<void> {
   ];
 
   while (true) {
+    let previousResult = false;
     console.log(`Running e2e tests with `)
-    await runTests(command, args, slackWebHookUrl)
+    previousResult = await runTests(command, args, slackWebHookUrl, previousResult)
     console.log(`Run done, sleeping for ${TIMEOUT_SECONDS} seconds`)
     await new Promise((resolve) => setTimeout(resolve, TIMEOUT_SECONDS * 1000));
   }
@@ -46,44 +47,52 @@ async function runTests(
   command: string,
   args: string[],
   slackWebHookUrl: string,
-): Promise<void> {
+  previousResult: boolean
+): Promise<boolean> {
+  let isSuccess = false;
+  let stdoutData = '';
   try {
-    await new Promise((resolve, reject) => {
-      const childProcess = spawn(command, args, {shell: true});
-
-      let stdoutData = '';
-      let stderrData = '';
+    isSuccess = await new Promise((resolve) => {
+      const childProcess = spawn(command, args, { shell: true });
 
       childProcess.stdout.on('data', (data) => {
-        console.log(data.toString())
+        stdoutData += data.toString();
+        console.log(data.toString());
       });
 
       childProcess.stderr.on('data', (data) => {
-        stderrData += data.toString();
+        console.error(data.toString());
       });
 
       childProcess.on('close', (code) => {
         if (code === 0) {
-          resolve(stdoutData);
+          resolve(true);
         } else {
-          reject(new Error(stderrData));
+          resolve(false);
         }
       });
-    })
+    });
   } catch (e: any) {
-    console.error(e.message)
+    console.error(e.message);
     await postSlackMessage(
-      `e2e blockchain tests failed\n` +
-      "```" +
-      `${e.message}` +
-      "```",
+      e.message,
       slackWebHookUrl,
-    )
+    );
   }
+
+  // If isSuccess is false, you might want to send a message to Slack here as well
+  if (!isSuccess || !previousResult) {
+    await postSlackMessage(
+      stdoutData,
+      slackWebHookUrl,
+    );
+  }
+
+  return isSuccess;
 }
 
 async function postSlackMessage(
-  text: string,
+  output: string,
   webhookUrl: string,
 ) {
   try {
@@ -95,7 +104,15 @@ async function postSlackMessage(
           "Content-type": "application/json"
         },
         body: JSON.stringify({
-          text
+          blocks: [
+            {
+              type: "section",
+              text: {
+                type: "mrkdwn",
+                text: "```" + output +"```"
+              }
+            }
+          ]
         })
       }
     )