diff --git a/contracts/tasks/e2e.ts b/contracts/tasks/e2e.ts index 5dc3deb..ff62bd5 100644 --- a/contracts/tasks/e2e.ts +++ b/contracts/tasks/e2e.ts @@ -19,39 +19,46 @@ task("e2e", "Runs all e2e tests") const contractAddress = taskArgs.contractAddress; const oracleAddress = taskArgs.oracleAddress; - await runOpenAi( + process.env.RUN_MODE = "e2e-script"; + + const testResults: Record = {}; + + let result = await runOpenAi( contractAddress, "gpt-4-turbo-preview", "Who is the president of USA?", hre, ) - await runOpenAi( + testResults["OpenAI gpt-4-turbo-preview"] = result.error || "✅"; + result = await runOpenAi( contractAddress, "gpt-3.5-turbo-1106", "Who is the president of USA?", hre, ) - await runGroq( + testResults["OpenAI gpt-3.5-turbo-1106"] = result.error || "✅"; + result = await runGroq( contractAddress, "llama2-70b-4096", "Who is the president of USA?", hre, ) - await runGroq( + testResults["Groq llama2-70b-4096"] = result.error || "✅"; + result = await runGroq( contractAddress, "mixtral-8x7b-32768", "Who is the president of USA?", hre, ) - await runGroq( + testResults["Groq mixtral-8x7b-32768"] = result.error || "✅"; + result = await runGroq( contractAddress, "gemma-7b-it", "Who is the president of USA?", hre, ) - - console.log(`Running "image_generation"`) - await runTaskWithTimeout( + testResults["gemma-7b-it mixtral-8x7b-32768"] = result.error || "✅"; + result = await runTaskWithTimeout( "image_generation", { contractAddress, @@ -59,10 +66,8 @@ task("e2e", "Runs all e2e tests") }, hre, ) - console.log(`DONE Running "image_generation"`) - - console.log(`Running "web_search"`) - await runTaskWithTimeout( + testResults["OpenAI image_generation"] = result.error || "✅"; + result = await runTaskWithTimeout( "web_search", { contractAddress, @@ -70,10 +75,8 @@ task("e2e", "Runs all e2e tests") }, hre, ) - console.log(`DONE Running "web_search"`) - - console.log(`Running "code_interpreter"`) - await runTaskWithTimeout( + testResults["web_search"] = result.error || "✅"; + result = await runTaskWithTimeout( "code_interpreter", { contractAddress, @@ -81,7 +84,7 @@ task("e2e", "Runs all e2e tests") }, hre, ) - console.log(`DONE Running "code_interpreter"`) + testResults["code_interpreter"] = result.error || "✅"; // console.log(`Running "add_knowledge_base"`) // await runTaskWithTimeout( @@ -93,8 +96,7 @@ task("e2e", "Runs all e2e tests") // hre, // ) // console.log(`DONE Running "add_knowledge_base"`) - console.log(`Running "query_knowledge_base"`) - await runTaskWithTimeout( + result = await runTaskWithTimeout( "query_knowledge_base", { contractAddress, @@ -103,18 +105,27 @@ task("e2e", "Runs all e2e tests") }, hre, ) - console.log(`DONE Running "query_knowledge_base"`) - - - console.log("================================================") - console.log(green, "e2e run done", reset) + testResults["query_knowledge_base"] = result.error || "✅"; + const totalTests = Object.keys(testResults).length; + const passedTests = Object.values(testResults).filter(result => result === "✅").length; + const failedTests = totalTests - passedTests; + console.log(`${passedTests} out of ${totalTests} tests passed `); + const transformedResults = Object.entries(testResults).map(([testName, result]) => ({ + "Test": testName, + "Result": result + })); + + console.table(transformedResults); + if (failedTests > 0) { + process.exit(1); + } }); async function runTaskWithTimeout( taskIdentifier: string, taskArguments: any, hre: HardhatRuntimeEnvironment, -) { +): Promise { try { const timeoutPromise = new Promise((resolve, reject) => { const id = setTimeout(() => { @@ -123,15 +134,15 @@ async function runTaskWithTimeout( }, TIMEOUT_SECONDS * 1000); }); - await Promise.race([ + const taskResult = await Promise.race([ timeoutPromise, hre.run(taskIdentifier, taskArguments), ]); + return taskResult; } catch (e: any) { process.stderr.write(e.message + " ") throw e } - } @@ -141,8 +152,7 @@ async function runOpenAi( message: string, hre: HardhatRuntimeEnvironment, ) { - console.log(`Running "openai", with model: ${model}`) - await runTaskWithTimeout( + let result = await runTaskWithTimeout( "openai", { contractAddress, @@ -151,7 +161,7 @@ async function runOpenAi( }, hre, ) - console.log(`DONE Running "openai", with model: ${model}.`) + return result; } async function runGroq( @@ -160,8 +170,7 @@ async function runGroq( message: string, hre: HardhatRuntimeEnvironment, ) { - console.log(`Running "groq", with model: ${model}`) - await runTaskWithTimeout( + let result = await runTaskWithTimeout( "groq", { contractAddress, @@ -170,5 +179,5 @@ async function runGroq( }, hre, ) - console.log(`DONE Running "groq", with model: ${model}.`) -} \ No newline at end of file + return result; +} diff --git a/contracts/tasks/e2eCron.ts b/contracts/tasks/e2eCron.ts index c701b4c..5aac1ba 100644 --- a/contracts/tasks/e2eCron.ts +++ b/contracts/tasks/e2eCron.ts @@ -34,9 +34,10 @@ async function main(): Promise { "--network", network ]; + let previousResult = false; while (true) { console.log(`Running e2e tests with `) - await runTests(command, args, slackWebHookUrl) + previousResult = await runTests(command, args, slackWebHookUrl, previousResult) console.log(`Run done, sleeping for ${TIMEOUT_SECONDS} seconds`) await new Promise((resolve) => setTimeout(resolve, TIMEOUT_SECONDS * 1000)); } @@ -46,44 +47,52 @@ async function runTests( command: string, args: string[], slackWebHookUrl: string, -): Promise { + previousResult: boolean +): Promise { + let isSuccess = false; + let stdoutData = ''; try { - await new Promise((resolve, reject) => { - const childProcess = spawn(command, args, {shell: true}); - - let stdoutData = ''; - let stderrData = ''; + isSuccess = await new Promise((resolve) => { + const childProcess = spawn(command, args, { shell: true }); childProcess.stdout.on('data', (data) => { - console.log(data.toString()) + stdoutData += data.toString(); + console.log(data.toString()); }); childProcess.stderr.on('data', (data) => { - stderrData += data.toString(); + console.error(data.toString()); }); childProcess.on('close', (code) => { if (code === 0) { - resolve(stdoutData); + resolve(true); } else { - reject(new Error(stderrData)); + resolve(false); } }); - }) + }); } catch (e: any) { - console.error(e.message) + console.error(e.message); await postSlackMessage( - `e2e blockchain tests failed\n` + - "```" + - `${e.message}` + - "```", + e.message, slackWebHookUrl, - ) + ); } + + // If isSuccess is false, you might want to send a message to Slack here as well + if (!isSuccess || !previousResult) { + await postSlackMessage( + stdoutData, + slackWebHookUrl, + ); + } + + return isSuccess; } async function postSlackMessage( - text: string, + output: string, webhookUrl: string, ) { try { @@ -95,7 +104,15 @@ async function postSlackMessage( "Content-type": "application/json" }, body: JSON.stringify({ - text + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: "```" + output +"```" + } + } + ] }) } ) diff --git a/contracts/tasks/functions.ts b/contracts/tasks/functions.ts index 99939ef..0f800cf 100644 --- a/contracts/tasks/functions.ts +++ b/contracts/tasks/functions.ts @@ -18,10 +18,7 @@ task("openai", "Calls the OpenAI LLM") const contract = await getContract("Test", contractAddress, hre); const response = await queryOpenAiLLM(contract, model, message, hre); - console.log(response); - if (response.error.length > 0) { - process.exit(1); - } + return checkResult(response); }); task("groq", "Calls the Groq LLM") @@ -35,10 +32,7 @@ task("groq", "Calls the Groq LLM") const contract = await getContract("Test", contractAddress, hre); const response = await queryGroqLLM(contract, model, message, hre); - console.log(response) - if (response.error.length > 0) { - process.exit(1); - } + return checkResult(response); }); task("web_search", "Calls the web search function") @@ -50,10 +44,7 @@ task("web_search", "Calls the web search function") const contract = await getContract("Test", contractAddress, hre); const response = await queryContractFunction(contract, "web_search", query, hre); - console.log(response) - if (response.error.length > 0) { - process.exit(1); - } + return checkResult(response); }); task("image_generation", "Calls the image generation function") @@ -65,10 +56,7 @@ task("image_generation", "Calls the image generation function") const contract = await getContract("Test", contractAddress, hre); const response = await queryContractFunction(contract, "image_generation", query, hre); - console.log(response) - if (response.error.length > 0) { - process.exit(1); - } + return checkResult(response); }); task("code_interpreter", "Calls the code interpreter function") @@ -80,10 +68,7 @@ task("code_interpreter", "Calls the code interpreter function") const contract = await getContract("Test", contractAddress, hre); const response = await queryContractFunction(contract, "code_interpreter", query, hre); - console.log(response) - if (response.error.length > 0) { - process.exit(1); - } + return checkResult(response); }); task("add_knowledge_base", "Adds a knowledge base to the contract") @@ -106,10 +91,7 @@ task("add_knowledge_base", "Adds a knowledge base to the contract") response = await contract.kbIndexes(cid); error = await contract.kbIndexingRequestErrors(runId); } - console.log({ response: response, error: error }); - if (error.length > 0) { - process.exit(1); - } + return checkResult({ response: response, error: error }); }); task("query_knowledge_base", "Queries a knowledge base") @@ -123,10 +105,7 @@ task("query_knowledge_base", "Queries a knowledge base") const contract = await getContract("Test", contractAddress, hre); const response = await queryContractKnowledgeBase(contract, cid, query, hre); - console.log(response) - if (response.error.length > 0) { - process.exit(1); - } + return checkResult(response); }); async function getContract( @@ -148,21 +127,18 @@ async function queryOpenAiLLM( try { const txResponse = await contract.callOpenAiLLM(model, message); await txResponse.wait(); - process.stdout.write("Waiting for response"); let response = await contract.lastResponse(); let error = await contract.lastError(); while (response.length === 0 && error.length === 0) { await new Promise((resolve) => setTimeout(resolve, 1000)); response = await contract.lastResponse(); error = await contract.lastError(); - process.stdout.write("."); } - console.log(""); return { response: response, error: error }; } catch (error) { console.error(`Error calling contract function: ${error}`); } - return { response: "", error: "Failed XX" }; + return { response: "", error: "Call failed" }; } async function queryGroqLLM( @@ -174,21 +150,18 @@ async function queryGroqLLM( try { const txResponse = await contract.callGroqLLM(model, message); await txResponse.wait(); - process.stdout.write("Waiting for response"); let response = await contract.lastResponse(); let error = await contract.lastError(); while (response.length === 0 && error.length === 0) { await new Promise((resolve) => setTimeout(resolve, 1000)); response = await contract.lastResponse(); error = await contract.lastError(); - process.stdout.write("."); } - console.log(""); return { response: response, error: error }; } catch (error) { console.error(`Error calling contract function: ${error}`); } - return { response: "", error: "Failed XX" }; + return { response: "", error: "Call failed" }; } async function queryContractFunction( @@ -200,21 +173,18 @@ async function queryContractFunction( try { const txResponse = await contract.callFunction(tool, query); await txResponse.wait(); - process.stdout.write("Waiting for response"); let response = await contract.lastResponse(); let error = await contract.lastError(); while (response.length === 0 && error.length === 0) { await new Promise((resolve) => setTimeout(resolve, 1000)); response = await contract.lastResponse(); error = await contract.lastError(); - process.stdout.write("."); } - console.log(""); return { response: response, error: error }; } catch (error) { console.error(`Error calling contract function: ${error}`); } - return { response: "", error: "Failed XX" }; + return { response: "", error: "Call failed" }; } async function queryContractKnowledgeBase( @@ -226,21 +196,18 @@ async function queryContractKnowledgeBase( try { const txResponse = await contract.queryKnowledgeBase(cid, query); await txResponse.wait(); - process.stdout.write("Waiting for response"); let response = await contract.lastResponse(); let error = await contract.lastError(); while (response.length === 0 && error.length === 0) { await new Promise((resolve) => setTimeout(resolve, 1000)); response = await contract.lastResponse(); error = await contract.lastError(); - process.stdout.write("."); } - console.log(""); return { response: response, error: error }; } catch (error) { console.error(`Error calling contract function: ${error}`); } - return { response: "", error: "Failed XX" }; + return { response: "", error: "Call failed" }; } function getRunId( @@ -263,4 +230,14 @@ function getRunId( } } return runId; +} + +function checkResult(result: FunctionResponse) : FunctionResponse { + if (process.env.RUN_MODE != "e2e-script") { + console.log(result) + if (result.error.length > 0) { + process.exit(1); + } + } + return result; } \ No newline at end of file