From 011ba8e09c76fce4435bdec3c36059f923fc9a88 Mon Sep 17 00:00:00 2001 From: Himanshu Ranjan Date: Wed, 3 Jan 2024 16:05:23 +0530 Subject: [PATCH 1/6] added failed test retry --- commands/generate_reports.js | 61 ++++++++---- commands/utils/batch/batch_runner.js | 140 +++++++++++++++++++++++++-- commands/utils/poller/build_stats.js | 1 + commands/utils/poller/poller.js | 13 +-- commands/utils/set_args.js | 10 ++ index.js | 9 +- 6 files changed, 201 insertions(+), 33 deletions(-) diff --git a/commands/generate_reports.js b/commands/generate_reports.js index 82463aa..a769398 100644 --- a/commands/generate_reports.js +++ b/commands/generate_reports.js @@ -17,6 +17,7 @@ function download_artefact( ) { return new Promise(function (resolve, reject) { let response_code; + let resp; if (!fs.existsSync(file_path)) { fs.mkdirSync(file_path, { recursive: true }); } @@ -44,6 +45,7 @@ function download_artefact( reject(err); } response_code = res.statusCode; + resp = res }).pipe( fs .createWriteStream(file_path, { @@ -65,6 +67,13 @@ function download_artefact( }); } else { fs.unlinkSync(file_path); + if (resp.body != null) { + const responseObject = JSON.parse(resp.body); + const dataValue = responseObject.data; + // console.log("Could not download artefacts with reason " + dataValue+ " for testID " + test_id) + reject("Could not download artefacts for test id " + test_id + " with reason " + dataValue); + } + // console.log("Could not download artefacts with reason " + resp.body + " for testID " + test_id) reject("Could not download artefacts for test id " + test_id); } }) @@ -73,7 +82,7 @@ function download_artefact( } function generate_report(args) { - return new Promise(function (resolve, reject) { + return new Promise(async function (resolve, reject) { var username = ""; var access_key = ""; @@ -96,7 +105,7 @@ function generate_report(args) { } else { reject("Access Key not provided"); } - + console.log("Generating report - step 3") //Check for session id if ( !("session_id" in args) || @@ -123,7 +132,7 @@ function generate_report(args) { ); } } - + console.log("Generating report - step 3") //set working enviornment var env = "prod"; if ("env" in args) { @@ -184,8 +193,11 @@ function generate_report(args) { fs.mkdirSync(directory, { recursive: true }); console.log("Directory created ", directory); } + console.log("Generating report - step 4") + const downloadPromises = []; + for (i = 0; i < build_info["data"].length; i++) { - download_artefact( + const downloadPromise = download_artefact( username, access_key, env, @@ -198,15 +210,32 @@ function generate_report(args) { ), build_payload["run_settings"]["reject_unauthorized"] ) - .then(function (resp) { - //Files downloaded - console.log(resp); - }) - .catch(function (err) { - console.log(err); - }); + downloadPromises.push(downloadPromise) + console.log("Generating report - step 5") } - resolve("Done"); + // wait here till all the files are downloaded + // await Promise.all(downloadPromises); + + Promise.allSettled(downloadPromises) + .then((results) => { + // results is an array of objects + for (const result of results) { + if (result.status == 'fulfilled') { + console.log(result.value); + } else if (result.status == 'rejected') { + console.log(result.reason); + } + } + resolve("Done"); + }) + .catch((error) => { + // This catch block will not be executed + console.log(error); + resolve("Done"); + }); + + // resolve("Done"); + console.log("Generating report - step 6") }) .catch(function (err) { console.log("Error occured while getting the build response", err); @@ -214,10 +243,6 @@ function generate_report(args) { }); } -module.exports = function (args) { - generate_report(args) - .then(function (resp) {}) - .catch(function (err) { - console.log("ERR:", err); - }); +module.exports = { + generate_report:generate_report }; diff --git a/commands/utils/batch/batch_runner.js b/commands/utils/batch/batch_runner.js index a8ed874..79c239a 100644 --- a/commands/utils/batch/batch_runner.js +++ b/commands/utils/batch/batch_runner.js @@ -12,6 +12,9 @@ const { del } = require("request"); const { delete_archive } = require("../archive.js"); const poller = require("../poller/poller.js"); const builds = require("../poller/build"); +const batcher = require("./batcher.js"); +const reports = require("../../../commands/generate_reports.js"); +const { fail } = require("yargs"); var batchCounter = 0; var totalBatches = 0; @@ -91,6 +94,7 @@ async function run(lt_config, batches, env) { .archive_project(lt_config) .then(function (file_obj) { project_file = file_obj["name"]; + lt_config["run_settings"]["project_file"] = project_file; //upload the project and get the project link uploader .upload_zip(lt_config, file_obj["name"], "project", env) @@ -121,14 +125,15 @@ async function run(lt_config, batches, env) { access_key: lt_config["lambdatest_auth"]["access_key"], type: "cypress" }); - run_test( payload, env, lt_config.run_settings.reject_unauthorized ) .then(function (session_id) { - delete_archive(project_file); + if (lt_config["run_settings"]["retry_failed"] == false ) { + delete_archive(project_file); + } delete_archive(file_obj["name"]); //listen to control+c signal and stop tests process.on("SIGINT", async () => { @@ -148,13 +153,48 @@ async function run(lt_config, batches, env) { }); if ( lt_config["run_settings"]["sync"] == true || - (lt_config["tunnel_settings"]["tunnel"] && lt_config["tunnel_settings"]["autostart"]) + (lt_config["tunnel_settings"]["tunnel"] && lt_config["tunnel_settings"]["autostart"]) || (lt_config["run_settings"]["retry_failed"] == true ) ) { console.log("Waiting for build to finish..."); - poller - .poll_build(lt_config, session_id, env) - .then(function (exit_code) { - resolve(exit_code); + poller.update_status(true); + poller.poll_build(lt_config, session_id, env) + .then( function (result) { + const { exit_code, build_info } = result; + if (lt_config["run_settings"]["retry_failed"] == true && build_info != null ) { + let failed_tests = []; + for (i = 0; i < build_info["data"].length; i++) { + if (build_info["data"][i]["status_ind"] == "failed" ) { + failed_tests.push(build_info["data"][i]["path"]); + } + } + if (failed_tests.length > 0) { + console.log("retrying these failed tests "+ failed_tests) + lt_config["run_settings"]["specs"]=failed_tests; + batcher + .make_batches(lt_config) + .then(function (batches) { + retry_run(lt_config, batches, env) + .then(function (exit_code) { + if (exit_code) { + console.log("retried failed tests ended with exit code " + exit_code); + } + resolve(exit_code); + }) + .catch(function (error) { + console.log(error); + resolve(1); + }); + }) + .catch(function (err) { + console.log(err); + resolve(1); + }); + } else { + resolve(exit_code); + } + } else { + resolve(exit_code); + } }) .catch(function (err) { console.log( @@ -193,6 +233,92 @@ async function run(lt_config, batches, env) { }); } +async function retry_run(lt_config, batches, env) { + totalBatches = batches.length; + return new Promise(function (resolve, reject) { + lt_config["test_suite"] = batches[0]; + archive + .archive_batch(lt_config, batches[0], env) + .then(async function (file_obj) { + uploader + .upload_zip(lt_config, file_obj["name"], "tests", env) + .then(async function (resp) { + + var payload = JSON.stringify({ + payload: { + test_file: resp["value"]["message"].split("?")[0], + }, + username: lt_config["lambdatest_auth"]["username"], + access_key: lt_config["lambdatest_auth"]["access_key"], + type: "cypress" + }); + + run_test( + payload, + env, + lt_config.run_settings.reject_unauthorized + ).then(function (session_id) { + + delete_archive(lt_config["run_settings"]["project_file"]); + delete_archive(file_obj["name"]); + + process.on("SIGINT", async () => { + try { + console.log( + "Retry - Control+c signal received.\nTrying to Terminate the processes" + ); + await builds.stop_cypress_session( + lt_config, + session_id, + env + ); + resolve(0); + } catch (e) { + console.log("Retry - Could not exit process. Try Again!!!"); + } + }); + if ( + lt_config["run_settings"]["sync"] == true || + (lt_config["tunnel_settings"]["tunnel"] && lt_config["tunnel_settings"]["autostart"]) + ) { + console.log("Retry - Waiting for build to finish..."); + poller.update_status(true); + poller.poll_build(lt_config, session_id, env) + .then(function (result) { + const { exit_code, build_json } = result; + resolve(exit_code); + }) + .catch(function (err) { + console.log( + "Retry - Some error occured in getting build updates", + err.message + ); + }); + } else { + resolve(0); + } + + }) + .catch(function (err) { + console.log("Retry - Error occured while creating tests", err); + }); + + + }) + .catch(function (err) { + console.log("Retry - Not able to archive the batch of test files", err); + }); + + }) + .catch(function (err) { + console.log("Retry - Unable to archive the project"); + console.log(err); + reject(err); + }); + }); +} + module.exports = { run_batches: run, + run_batches_retry: retry_run, }; diff --git a/commands/utils/poller/build_stats.js b/commands/utils/poller/build_stats.js index 23fda4b..0e4aa4f 100644 --- a/commands/utils/poller/build_stats.js +++ b/commands/utils/poller/build_stats.js @@ -106,6 +106,7 @@ function get_build_info(lt_config, session_id, env, update_status, callback) { ) { get_build_info_count = get_build_info_count + 1; if (get_build_info_count > 4) { + get_build_info_count = 0; update_status(false); return callback(null, JSON.parse(body)); } diff --git a/commands/utils/poller/poller.js b/commands/utils/poller/poller.js index 2e1b034..65f78e5 100644 --- a/commands/utils/poller/poller.js +++ b/commands/utils/poller/poller.js @@ -23,10 +23,10 @@ function poll_build(lt_config, session_id, env) { if (err == null) { build_stats .get_completed_build_info(lt_config, session_id, env) - .then(function (build_info) { + .then(async function (build_info) { if (!build_info || build_info.data == null) { console.log("Build info not found"); - resolve(1); + resolve({exit_code:1, build_info:build_info}); return; } let stats = {}; @@ -55,16 +55,17 @@ function poll_build(lt_config, session_id, env) { reject_unauthorized: lt_config.run_settings.reject_unauthorized, }; - reports(args); + + await reports.generate_report(args) } if ( Object.keys(stats).length == 1 && (Object.keys(stats).includes("completed") || Object.keys(stats).includes("passed")) ) { - resolve(0); + resolve({exit_code:0, build_info:build_info}); } else { - resolve(1); + resolve({exit_code:1, build_info:build_info}); } }) .catch(function (err) { @@ -72,7 +73,7 @@ function poll_build(lt_config, session_id, env) { }); } else { console.log(err); - resolve(1); + resolve({exit_code:1, build_info:null}); } } ); diff --git a/commands/utils/set_args.js b/commands/utils/set_args.js index 401a513..6581773 100644 --- a/commands/utils/set_args.js +++ b/commands/utils/set_args.js @@ -387,6 +387,16 @@ function sync_args_from_cmd(args) { lt_config["run_settings"]["network_sse"] = false; } + if ("retry_failed" in args) { + if (args["retry_failed"] == "true") { + lt_config.run_settings.retry_failed = true; + } else { + lt_config.run_settings.retry_failed = false; + } + } else if (lt_config["run_settings"]["retry_failed"] && !lt_config["run_settings"]["retry_failed"]) { + lt_config["run_settings"]["retry_failed"] = false; + } + if ("headless" in args) { lt_config["run_settings"]["headless"] = args["headless"]; } else if (!lt_config["run_settings"]["headless"]) { diff --git a/index.js b/index.js index 983d53e..ecc87bd 100644 --- a/index.js +++ b/index.js @@ -213,6 +213,11 @@ const argv = require("yargs") describe: "show command logs on dashboard.", type: "string", }) + .option("ret_fail", { + alias: "retry_failed", + describe: "run failed tests in a new build.", + type: "bool", + }) .option("net_http2", { alias: "network_http2", describe: "Capture Http2 Network logs", @@ -227,7 +232,7 @@ const argv = require("yargs") alias: "network_sse", describe: "Bypass sse events calls for Network logs", type: "bool", - });; + }); }, function (argv) { require("./commands/run")(argv); @@ -344,7 +349,7 @@ const argv = require("yargs") }); }, function (argv) { - require("./commands/generate_reports")(argv); + require("./commands/generate_reports").generate_report(argv); } ) .help().argv; From b4fafb2259609231917099e4ff2875026deff92b Mon Sep 17 00:00:00 2001 From: Himanshu Ranjan Date: Wed, 3 Jan 2024 16:07:27 +0530 Subject: [PATCH 2/6] removed comments --- commands/generate_reports.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/commands/generate_reports.js b/commands/generate_reports.js index a769398..0dc421d 100644 --- a/commands/generate_reports.js +++ b/commands/generate_reports.js @@ -70,10 +70,8 @@ function download_artefact( if (resp.body != null) { const responseObject = JSON.parse(resp.body); const dataValue = responseObject.data; - // console.log("Could not download artefacts with reason " + dataValue+ " for testID " + test_id) reject("Could not download artefacts for test id " + test_id + " with reason " + dataValue); } - // console.log("Could not download artefacts with reason " + resp.body + " for testID " + test_id) reject("Could not download artefacts for test id " + test_id); } }) @@ -105,7 +103,6 @@ function generate_report(args) { } else { reject("Access Key not provided"); } - console.log("Generating report - step 3") //Check for session id if ( !("session_id" in args) || @@ -132,7 +129,6 @@ function generate_report(args) { ); } } - console.log("Generating report - step 3") //set working enviornment var env = "prod"; if ("env" in args) { @@ -193,7 +189,6 @@ function generate_report(args) { fs.mkdirSync(directory, { recursive: true }); console.log("Directory created ", directory); } - console.log("Generating report - step 4") const downloadPromises = []; for (i = 0; i < build_info["data"].length; i++) { @@ -211,10 +206,7 @@ function generate_report(args) { build_payload["run_settings"]["reject_unauthorized"] ) downloadPromises.push(downloadPromise) - console.log("Generating report - step 5") } - // wait here till all the files are downloaded - // await Promise.all(downloadPromises); Promise.allSettled(downloadPromises) .then((results) => { @@ -234,8 +226,6 @@ function generate_report(args) { resolve("Done"); }); - // resolve("Done"); - console.log("Generating report - step 6") }) .catch(function (err) { console.log("Error occured while getting the build response", err); From c0ee3e96656d4f3788a72581ae8cfc1f3d9b8b6b Mon Sep 17 00:00:00 2001 From: Himanshu Ranjan Date: Wed, 3 Jan 2024 18:19:05 +0530 Subject: [PATCH 3/6] fix undefined error when lambda auth is not present in config file --- commands/utils/set_args.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/commands/utils/set_args.js b/commands/utils/set_args.js index 6581773..7658576 100644 --- a/commands/utils/set_args.js +++ b/commands/utils/set_args.js @@ -111,6 +111,9 @@ function sync_args_from_cmd(args) { } lt_config["lambdatest_auth"]["username"] = process.env.LT_USERNAME; } else if ("username" in args && args["username"] != "") { + if (!lt_config["lambdatest_auth"]) { + lt_config["lambdatest_auth"] = {}; + } lt_config["lambdatest_auth"]["username"] = args["username"]; } @@ -143,6 +146,9 @@ function sync_args_from_cmd(args) { console.log("Setting access key from environment"); lt_config["lambdatest_auth"]["access_key"] = process.env.LT_ACCESS_KEY; } else if ("access_key" in args && args["access_key"] != "") { + if (!lt_config["lambdatest_auth"]) { + lt_config["lambdatest_auth"] = {}; + } lt_config["lambdatest_auth"]["access_key"] = args["access_key"]; } From 09990d798ccd831473c72505fc2b485912b7ed86 Mon Sep 17 00:00:00 2001 From: Himanshu Ranjan Date: Sat, 13 Jan 2024 19:13:22 +0530 Subject: [PATCH 4/6] download artefacts only when sync flag is true. --- commands/utils/batch/batch_runner.js | 2 +- commands/utils/poller/poller.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/utils/batch/batch_runner.js b/commands/utils/batch/batch_runner.js index 79c239a..5ebb6f8 100644 --- a/commands/utils/batch/batch_runner.js +++ b/commands/utils/batch/batch_runner.js @@ -168,7 +168,7 @@ async function run(lt_config, batches, env) { } } if (failed_tests.length > 0) { - console.log("retrying these failed tests "+ failed_tests) + console.log("Retrying failed tests.") lt_config["run_settings"]["specs"]=failed_tests; batcher .make_batches(lt_config) diff --git a/commands/utils/poller/poller.js b/commands/utils/poller/poller.js index 65f78e5..6c82a24 100644 --- a/commands/utils/poller/poller.js +++ b/commands/utils/poller/poller.js @@ -46,7 +46,7 @@ function poll_build(lt_config, session_id, env) { console.table(status); console.log(stats); //Download the artefacts if downloads is passed - if (lt_config.run_settings.downloads != "") { + if (lt_config.run_settings.downloads != "" && lt_config["run_settings"]["sync"] == true) { let args = { user: lt_config.lambdatest_auth.username, access_key: lt_config.lambdatest_auth.access_key, From b55ede359540b1d7b0eadef46de5e1044ffa673d Mon Sep 17 00:00:00 2001 From: Himanshu Ranjan Date: Wed, 17 Jan 2024 19:57:27 +0530 Subject: [PATCH 5/6] fixed failed test creation logic --- commands/utils/batch/batch_runner.js | 33 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/commands/utils/batch/batch_runner.js b/commands/utils/batch/batch_runner.js index 5ebb6f8..d73e997 100644 --- a/commands/utils/batch/batch_runner.js +++ b/commands/utils/batch/batch_runner.js @@ -161,18 +161,23 @@ async function run(lt_config, batches, env) { .then( function (result) { const { exit_code, build_info } = result; if (lt_config["run_settings"]["retry_failed"] == true && build_info != null ) { - let failed_tests = []; + let failed_test_suites = []; for (i = 0; i < build_info["data"].length; i++) { - if (build_info["data"][i]["status_ind"] == "failed" ) { - failed_tests.push(build_info["data"][i]["path"]); + if (build_info["data"][i]["status_ind"] == "failed") { + let failed_spec = findSpecFile(lt_config["test_suite"],build_info["data"][i]) + let failed_suite = { + spec_file: failed_spec, + path: build_info["data"][i]["path"], + browser: build_info["data"][i]["browser"], + version: build_info["data"][i]["version"], + platform: build_info["data"][i]["platform"] + } + failed_test_suites.push(failed_suite); } } - if (failed_tests.length > 0) { + if (failed_test_suites.length > 0) { console.log("Retrying failed tests.") - lt_config["run_settings"]["specs"]=failed_tests; - batcher - .make_batches(lt_config) - .then(function (batches) { + let batches = [failed_test_suites] retry_run(lt_config, batches, env) .then(function (exit_code) { if (exit_code) { @@ -183,12 +188,7 @@ async function run(lt_config, batches, env) { .catch(function (error) { console.log(error); resolve(1); - }); - }) - .catch(function (err) { - console.log(err); - resolve(1); - }); + }); } else { resolve(exit_code); } @@ -318,6 +318,11 @@ async function retry_run(lt_config, batches, env) { }); } +function findSpecFile(testSuite, buildInfoData) { + const foundTest = testSuite.find((test) => test.path === buildInfoData.path); + return foundTest ? foundTest.spec_file : null; +} + module.exports = { run_batches: run, run_batches_retry: retry_run, From dc13d4db8381614d43cc712f8f8f2b88503b00e3 Mon Sep 17 00:00:00 2001 From: Himanshu Ranjan Date: Thu, 18 Jan 2024 13:48:06 +0530 Subject: [PATCH 6/6] bump version --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 03e212a..be89b3c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lambdatest-cypress-cli", - "version": "3.0.25", + "version": "3.0.26", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "lambdatest-cypress-cli", - "version": "3.0.25", + "version": "3.0.26", "license": "MIT", "dependencies": { "@lambdatest/node-tunnel": "latest", diff --git a/package.json b/package.json index 7827a96..e33d99a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lambdatest-cypress-cli", - "version": "3.0.25", + "version": "3.0.26", "description": "The lambdatest-cypress-cli is LambdaTest's command-line interface (CLI) aimed to help you run your Cypress tests on LambdaTest platform.", "homepage": "https://github.com/LambdaTest/lambdatest-cypress-cli", "author": "LambdaTest ",