diff --git a/.github/workflows/APITestsHardGate-Failing_gradle.yml b/.github/workflows/APITestsHardGate-Failing_gradle.yml new file mode 100644 index 0000000..3d45c1b --- /dev/null +++ b/.github/workflows/APITestsHardGate-Failing_gradle.yml @@ -0,0 +1,34 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: APITestsHardGate-Failing + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + - name: Build with Gradle and run unit tests + uses: gradle/gradle-build-action@e2097ccd7e8ed48671dc068ac4efa86d25745b39 + with: + arguments: build --refresh-dependencies + - name: Run failing API tests as gradle task with SET_HARD_GATE=true + run: | + TEST_TYPE=api TAGS=hardGate TARGET_ENVIRONMENT=prod SET_HARD_GATE=true IS_FAILING_TEST_SUITE=true ./gradlew clean run diff --git a/.github/workflows/APITestsHardGate-Passing_gradle.yml b/.github/workflows/APITestsHardGate-Passing_gradle.yml new file mode 100644 index 0000000..6184bc9 --- /dev/null +++ b/.github/workflows/APITestsHardGate-Passing_gradle.yml @@ -0,0 +1,34 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: APITestsHardGate-Passing + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + - name: Build with Gradle and run unit tests + uses: gradle/gradle-build-action@e2097ccd7e8ed48671dc068ac4efa86d25745b39 + with: + arguments: build --refresh-dependencies + - name: Run passing API tests as gradle task with SET_HARD_GATE=true + run: | + TEST_TYPE=api TAGS=hardGate TARGET_ENVIRONMENT=prod SET_HARD_GATE=true IS_FAILING_TEST_SUITE=false ./gradlew clean run diff --git a/.github/workflows/APITests_gradle.yml b/.github/workflows/APITests_gradle.yml index c610691..5b04db1 100644 --- a/.github/workflows/APITests_gradle.yml +++ b/.github/workflows/APITests_gradle.yml @@ -31,4 +31,4 @@ jobs: arguments: build --refresh-dependencies - name: Run API tests as gradle task run: | - TEST_TYPE=api TARGET_ENVIRONMENT=prod ./gradlew clean test + TEST_TYPE=api TARGET_ENVIRONMENT=prod ./gradlew clean run diff --git a/.github/workflows/WorkflowTests_gradle.yml b/.github/workflows/WorkflowTests_gradle.yml index 2042885..11cc9a6 100644 --- a/.github/workflows/WorkflowTests_gradle.yml +++ b/.github/workflows/WorkflowTests_gradle.yml @@ -31,4 +31,4 @@ jobs: arguments: build --refresh-dependencies - name: Run Workflow tests as gradle task run: | - TEST_TYPE=workflow TARGET_ENVIRONMENT=prod ./gradlew clean test + TEST_TYPE=workflow TARGET_ENVIRONMENT=prod ./gradlew clean run diff --git a/README.md b/README.md index 1d59dc4..b25e2e3 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Example ## [ReportPortal](./docs/ReportPortal.md) +## [Setting up the Hard Gate](./docs/HardGate.md) + ## [Machine setup](./docs/MachineSetup.md) ## Guidelines diff --git a/docs/HardGate.md b/docs/HardGate.md new file mode 100644 index 0000000..6446149 --- /dev/null +++ b/docs/HardGate.md @@ -0,0 +1,49 @@ +Back to main [README](./../README.md) + +----- + +# Setting up a Hard Gate with kendo + +kendo now offers the capability to establish a Hard Gate for your test execution results, ensuring precise control over your testing process. + +### **For passing tests, any failure within the executed scenarios results in the build being marked as FAILED.** + +### **For failing tests, any occurrence of a _passing_ scenario(s) automatically flags the build as FAILED.** + +This stringent control mechanism not only upholds the reliability of your testing process but also ensures that identified issues are accurately reflected in the build status, facilitating efficient debugging and resolution. + +By implementing the Hard Gate functionality in kendo, you can enhance the efficiency and effectiveness of your test management, enabling precise control and clear insights into your testing outcomes. + +## Enabling the Hard Gate + +To activate the Hard Gate feature, simply include the following parameter in your test execution command or set it as an environment variable: + + SET_HARD_GATE=true + +Once enabled, the Hard Gate provides distinct functionalities for both passing and failing tests. + +### Handling Passing Tests + +With the Hard Gate enabled, passing tests become the focus of execution. By default, all tests tagged with **@failing** are excluded from execution, ensuring that only non-failing tests, i.e., the ones expected to pass, are run. + +To explicitly disable the execution of failing tests, use the following parameter in your test execution command or set it as an environment variable: + + IS_FAILING_TEST_SUITE=false + +If any scenario marked as passing fails during execution, the build status will be marked as **FAILED**. + +## Dealing with Failing Tests + +In the context of the Hard Gate, failing tests are rigorously managed to ensure comprehensive evaluation. + +### Executing Failing Tests + +To specifically execute failing tests, utilize the following parameter in your test execution command: + + IS_FAILING_TEST_SUITE=true + +With this parameter, only tests tagged with **@failing** will be executed. If all scenarios marked as failing indeed fail during execution, the build will be marked as **PASS**. This provides clarity on whether a previously failing test now passes due to a fix in either the test itself or the underlying product issue. + +----- + +Back to main [README](./../README.md) diff --git a/src/main/java/com/znsio/kendo/FatJarRunner.java b/src/main/java/com/znsio/kendo/FatJarRunner.java index 8d7a1ca..9a2dc80 100644 --- a/src/main/java/com/znsio/kendo/FatJarRunner.java +++ b/src/main/java/com/znsio/kendo/FatJarRunner.java @@ -1,5 +1,6 @@ package com.znsio.kendo; +import com.znsio.kendo.exceptions.HardGateFailedException; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.TestPlan; @@ -39,17 +40,28 @@ public static void main(String[] args) { FatJarRunner runner = new FatJarRunner(); runner.runAll(); + boolean isFailTheBuild = Boolean.parseBoolean(System.getProperty(RunTest.Metadata.FAIL_THE_BUILD.name(), String.valueOf(false))); + TestExecutionSummary summary = runner.listener.getSummary(); summary.printTo(new PrintWriter(System.out)); - if (summary.getTestsAbortedCount() > 0 || summary.getTestsFailedCount() > 0) { - String testFailureSummary = ""; + + boolean haveTestsFailed = summary.getTestsAbortedCount() > 0 || summary.getTestsFailedCount() > 0; + String testFailureSummary = ""; + if (haveTestsFailed) { for (Failure failure : summary.getFailures()) { testFailureSummary += failure.getTestIdentifier() .getDisplayName() + "->\n" + failure.getException() .toString() + "\n"; } + } - throw new RuntimeException("Tests failed" + testFailureSummary); + if (isFailTheBuild) { + System.out.println("FatJarRunner: Hard gate failure - throwing exception"); + throw new HardGateFailedException("Hard gate failure" + testFailureSummary); + } else { + if (haveTestsFailed) { + throw new RuntimeException("Tests failed" + testFailureSummary); + } } } } diff --git a/src/main/java/com/znsio/kendo/RunTest.java b/src/main/java/com/znsio/kendo/RunTest.java index 0b53604..7255bd4 100644 --- a/src/main/java/com/znsio/kendo/RunTest.java +++ b/src/main/java/com/znsio/kendo/RunTest.java @@ -5,6 +5,7 @@ import com.jayway.jsonpath.JsonPath; import com.znsio.kendo.cmd.CommandLineExecutor; import com.znsio.kendo.cmd.CommandLineResponse; +import com.znsio.kendo.exceptions.HardGateFailedException; import com.znsio.kendo.exceptions.InvalidTestDataException; import com.znsio.kendo.exceptions.TestExecutionException; import org.junit.jupiter.api.Test; @@ -69,8 +70,10 @@ public class RunTest { private final String testDataFile; private final String rpEnable; private final String DEFAULT_CONFIG_FILE_NAME = "./src/test/java/config.properties"; + private final boolean isSetHardGate; + private final boolean isFailingTestSuite; - private enum Metadata { + enum Metadata { BRANCH_NAME_ENV_VAR, BUILD_ID_ENV_VAR, BUILD_INITIATION_REASON_ENV_VAR, @@ -81,6 +84,9 @@ private enum Metadata { TEST_DATA_FILE_NAME, TEST_TYPE, TARGET_ENVIRONMENT, + SET_HARD_GATE, + IS_FAILING_TEST_SUITE, + FAIL_THE_BUILD, CONFIG_FILE } @@ -104,6 +110,8 @@ public RunTest() { customTags = getOverloadedValueFromPropertiesFor(Metadata.TAGS, ""); testDataFile = getTestDataFileName(); testType = getTestType(); + isSetHardGate = Boolean.parseBoolean(getOverloadedValueFromPropertiesFor(Metadata.SET_HARD_GATE, String.valueOf(false))); + isFailingTestSuite = Boolean.parseBoolean(getOverloadedValueFromPropertiesFor(Metadata.IS_FAILING_TEST_SUITE, String.valueOf(false))); karateEnv = getOverloadedValueFromPropertiesFor(Metadata.TARGET_ENVIRONMENT, NOT_SET); String testDataFileName = new File(testDataFile).getName(); @@ -215,18 +223,59 @@ void runKarateTests() { .outputHtmlReport(true) .parallel(getParallelCount()); String reportFilePath = generateCucumberHtmlReport(results.getReportDir()); - String message = "\n\n" + "Test execution summary: "; - message += "\n\t" + "Tags: " + tags; - message += "\n\t" + "Environment: " + karateEnv; - message += "\n\t" + "Parallel count: " + getParallelCount(); - message += "\n\t" + "Scenarios: Failed: " + results.getScenariosFailed() + ", Passed: " + results.getScenariosPassed() + ", Total: " + results.getScenariosTotal(); - message += "\n\t" + "Features : Failed: " + results.getFeaturesFailed() + ", Passed: " + results.getFeaturesPassed() + ", Total: " + results.getFeaturesTotal(); - message += "\n\t" + "Karate Reports available here: file://" + karateReportsDir + File.separator + "karate-summary.html"; - message += "\n\t" + "Cucumber Reports available here: file://" + reportFilePath; - if (results.getScenariosFailed() > 0) { - throw new TestExecutionException(message); + logTestExecutionStatus result = getLogTestExecutionStatus(results, karateReportsDir, reportFilePath); + checkHardGate(result.scenariosPassed(), result.scenariosFailed(), result.testExecutionSummaryMessage()); + } + + private logTestExecutionStatus getLogTestExecutionStatus(Results results, String karateReportsDir, String reportFilePath) { + String testExecutionSummaryMessage = "\n\n" + "Test execution summary: "; + testExecutionSummaryMessage += "\n\t" + "Tags: " + tags; + testExecutionSummaryMessage += "\n\t" + "Environment: " + karateEnv; + testExecutionSummaryMessage += "\n\t" + "Parallel count: " + getParallelCount(); + int scenariosFailed = results.getScenariosFailed(); + int scenariosPassed = results.getScenariosPassed(); + int scenariosTotal = results.getScenariosTotal(); + testExecutionSummaryMessage += "\n\t" + "Scenarios: Failed: " + scenariosFailed + ", Passed: " + scenariosPassed + ", Total: " + scenariosTotal; + testExecutionSummaryMessage += "\n\t" + "Features : Failed: " + results.getFeaturesFailed() + ", Passed: " + results.getFeaturesPassed() + ", Total: " + results.getFeaturesTotal(); + testExecutionSummaryMessage += "\n\t" + "Karate Reports available here: file://" + karateReportsDir + File.separator + "karate-summary.html"; + testExecutionSummaryMessage += "\n\t" + "Cucumber Reports available here: file://" + reportFilePath; + + logTestExecutionStatus result = new logTestExecutionStatus(testExecutionSummaryMessage, scenariosFailed, scenariosPassed); + return result; + } + + private record logTestExecutionStatus(String testExecutionSummaryMessage, int scenariosFailed, int scenariosPassed) { + } + + private void checkHardGate(int scenariosPassed, int scenariosFailed, String testExecutionSummaryMessage) { + if (isSetHardGate) { + if (isFailingTestSuite) { + if (scenariosPassed == 0) { + // pass the build + System.out.printf("Hard Gate passed for Failing Tests build. %n%s%n", testExecutionSummaryMessage); + } else { + // hard gate exception + System.setProperty(RunTest.Metadata.FAIL_THE_BUILD.name(), String.valueOf(true)); + throw new HardGateFailedException("Hard Gate failed for Failing Tests build. %n%s".formatted(testExecutionSummaryMessage)); + } + } else { + if (scenariosFailed == 0) { + // pass the build + System.out.printf("Hard Gate passed for Passing Tests build. %n%s%n", testExecutionSummaryMessage); + } else { + // hard gate exception + System.setProperty(RunTest.Metadata.FAIL_THE_BUILD.name(), String.valueOf(true)); + throw new HardGateFailedException("Hard Gate failed for Passing Tests build. %n%s".formatted(testExecutionSummaryMessage)); + } + } } else { - System.out.println(message); + if (scenariosFailed > 0) { + System.out.println("Hard Gate not set. Test(s) failed."); + throw new TestExecutionException(testExecutionSummaryMessage); + } else { + System.out.println("Hard Gate not set. Test(s) passed."); + System.out.println(testExecutionSummaryMessage); + } } } @@ -420,6 +469,13 @@ private List getTags() { tagsToRun.add("~@wip"); tagsToRun.add("~@template"); tagsToRun.add("~@data"); + if (isSetHardGate) { + if (isFailingTestSuite) { + tagsToRun.add("@failing"); + } else { + tagsToRun.add("~@failing"); + } + } System.out.println("Run tests with tags: " + tagsToRun); return tagsToRun; diff --git a/src/main/java/com/znsio/kendo/exceptions/HardGateFailedException.java b/src/main/java/com/znsio/kendo/exceptions/HardGateFailedException.java new file mode 100644 index 0000000..a68c435 --- /dev/null +++ b/src/main/java/com/znsio/kendo/exceptions/HardGateFailedException.java @@ -0,0 +1,7 @@ +package com.znsio.kendo.exceptions; + +public class HardGateFailedException extends RuntimeException { + public HardGateFailedException(String failureMessage) { + super(failureMessage); + } +} diff --git a/src/test/java/com/znsio/api/hardGateTags.feature b/src/test/java/com/znsio/api/hardGateTags.feature new file mode 100644 index 0000000..ded9f40 --- /dev/null +++ b/src/test/java/com/znsio/api/hardGateTags.feature @@ -0,0 +1,51 @@ +@eat @demo @prod @tags @hardGate +Feature: tags api test + + @first + Scenario: first api + And print 'first api' + + @second @local @demo2 + Scenario: second api + And print 'second api' + And print 'system property foo in second api:', karate.properties['foo'] + + @second @e2e + Scenario: second e2e api + And print 'second e2e api' + + @second @e2e @wip @failing + Scenario: second e2e api failing wip + And print 'second e2e api wip' + + @second @e2e @failing + Scenario: second e2e api failing + And print 'second e2e api failing' + And match 1 == 2 + + @second @e2e @failing + Scenario: second e2e api failing skip + And print 'second e2e api failing skip' + And match 1 == 2 + And print 'second e2e api failing skip' + + @second @e2e @failing + Scenario Outline: second e2e api failing skip + And print 'second e2e api failing skip' + And match == + And print 'second e2e api failing skip' + Examples: + | num1 | num2 | + | 1 | 2 | + | 3 | 4 | + + @second @e2e + Scenario Outline: second e2e api passing skip + And print 'second e2e api passing skip' + And match == + And print 'second e2e api passing skip' + Examples: + | num1 | num2 | + | 1 | 1 | + | 2 | 2 | + | 3 | 3 | diff --git a/src/test/java/config.properties b/src/test/java/config.properties index 2b2a90f..03563f1 100644 --- a/src/test/java/config.properties +++ b/src/test/java/config.properties @@ -3,8 +3,10 @@ BUILD_ID_ENV_VAR=BUILD_ID BUILD_INITIATION_REASON_ENV_VAR=BUILD_REASON PARALLEL_COUNT=2 PROJECT_NAME=karate tests +RP_ENABLE=false TAGS= TEST_DATA_FILE_NAME=./src/test/java/test_data.json TEST_TYPE=api TARGET_ENVIRONMENT=prod -RP_ENABLE=false +SET_HARD_GATE=false +IS_FAILING_TEST_SUITE=false