From cdcbad71c18f815e0ff83cf5e8b7125249dcf6dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Gro=C3=9Fmann?= Date: Wed, 6 Nov 2024 11:08:33 +0100 Subject: [PATCH] Refactoring handling of skipped tests, updating log formatter --- .../finish/MethodContextUpdateWorker.java | 29 +++++---- .../testng/worker/finish/MethodEndWorker.java | 2 +- .../worker/start/MethodStartWorker.java | 11 ++-- .../report/TesterraListener.java | 65 ++++++++++--------- .../report/model/context/ClassContext.java | 22 +++---- .../tic/testframework/utils/Formatter.java | 32 +++++++-- .../core/utils/Log4jFileReader.java | 4 +- .../pretest_status/DataProvider2Test.java | 51 +++++++++++++++ .../FailedSetupMethodTests.java | 37 ----------- .../test/status/CheckTestStatusTest.java | 49 +++++++------- 10 files changed, 174 insertions(+), 128 deletions(-) create mode 100644 integration-tests/src/test/java/io/testerra/test/pretest_status/DataProvider2Test.java diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodContextUpdateWorker.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodContextUpdateWorker.java index d5c0603288..701d9ff920 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodContextUpdateWorker.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodContextUpdateWorker.java @@ -58,19 +58,19 @@ public void onMethodEnd(MethodEndEvent event) { testResult.setStatus(ITestResult.FAILURE); methodContext.setStatus(Status.FAILED); } - StringBuilder sb = new StringBuilder(); - sb.append("The following assertions failed in dataprovider method "); - sb.append(testResult.getMethod().getDataProviderMethod().getMethod().getName()); - sb.append(":"); - AtomicInteger i = new AtomicInteger(); - methodContext.readErrors() -// .filter(ErrorContext::isNotOptional) - .forEach(errorContext -> { - i.incrementAndGet(); - sb.append("\n").append(i).append(") ").append(errorContext.getThrowable().getMessage()); - }); - AssertionError testMethodContainerError = new AssertionError(sb.toString()); - testResult.setThrowable(testMethodContainerError); +// StringBuilder sb = new StringBuilder(); +// sb.append("The following assertions failed in dataprovider method "); +// sb.append(testResult.getMethod().getDataProviderMethod().getMethod().getName()); +// sb.append(":"); +// AtomicInteger i = new AtomicInteger(); +// methodContext.readErrors() +//// .filter(ErrorContext::isNotOptional) +// .forEach(errorContext -> { +// i.incrementAndGet(); +// sb.append("\n").append(i).append(") ").append(errorContext.getThrowable().getMessage()); +// }); +// AssertionError testMethodContainerError = new AssertionError(sb.toString()); +// testResult.setThrowable(testMethodContainerError); } else // Handle collected assertions if we have more than one if (testResult.isSuccess() && methodContext.readErrors().anyMatch(ErrorContext::isNotOptional)) { @@ -128,6 +128,9 @@ public void onMethodEnd(MethodEndEvent event) { }); TestStep failedStep = methodContext.getCurrentTestStep(); methodContext.setFailedStep(failedStep); + } else if (event.isSkipped()) { + // Can be caused by + methodContext.setStatus(Status.SKIPPED); } else if (testResult.isSuccess()) { methodContext.setStatus(Status.PASSED); RetryAnalyzer.methodHasBeenPassed(methodContext); diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodEndWorker.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodEndWorker.java index 9a34990a03..ab3273a397 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodEndWorker.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodEndWorker.java @@ -45,7 +45,7 @@ public void onMethodEnd(MethodEndEvent event) { ITestNGMethod testMethod = event.getTestMethod(); MethodContext methodContext = event.getMethodContext(); - String msg = String.format("%s %s", methodContext.getStatus().title, formatter.toString(testMethod)); + String msg = String.format("%s %s", methodContext.getStatus().title, formatter.toString(testResult)); if (event.isFailed()) { log().error(msg, testResult.getThrowable()); } else if (event.getTestResult().isSuccess()) { diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/start/MethodStartWorker.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/start/MethodStartWorker.java index 92138293b6..a01c48688b 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/start/MethodStartWorker.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/start/MethodStartWorker.java @@ -22,18 +22,17 @@ package eu.tsystems.mms.tic.testframework.execution.testng.worker.start; import com.google.common.eventbus.Subscribe; -import eu.tsystems.mms.tic.testframework.annotations.NoRetry; import eu.tsystems.mms.tic.testframework.common.Testerra; import eu.tsystems.mms.tic.testframework.events.MethodStartEvent; import eu.tsystems.mms.tic.testframework.execution.testng.RetryAnalyzer; import eu.tsystems.mms.tic.testframework.logging.Loggable; -import eu.tsystems.mms.tic.testframework.utils.DefaultFormatter; import eu.tsystems.mms.tic.testframework.utils.Formatter; -import java.lang.reflect.Method; import org.testng.IRetryAnalyzer; import org.testng.ITestNGMethod; import org.testng.internal.annotations.DisabledRetryAnalyzer; +import java.lang.reflect.Method; + public class MethodStartWorker implements Loggable, MethodStartEvent.Listener { private final Formatter formatter = Testerra.getInjector().getInstance(Formatter.class); @@ -46,7 +45,7 @@ public void onMethodStart(MethodStartEvent event) { addRetryAnalyzer(event); } - log().info("Run " + formatter.toString(event.getTestMethod())); + log().info("Run {}", formatter.toString(event.getTestResult())); } private void addRetryAnalyzer(MethodStartEvent event) { @@ -55,8 +54,8 @@ private void addRetryAnalyzer(MethodStartEvent event) { Method method = event.getMethod(); if (retryAnalyzer == null || retryAnalyzer instanceof DisabledRetryAnalyzer) { testNGMethod.setRetryAnalyzerClass(RetryAnalyzer.class); - } else if (!(retryAnalyzer instanceof RetryAnalyzer)){ - log().info("Using a non-default retry analyzer: " + retryAnalyzer.getClass().getSimpleName() + " on " + method.getName()); + } else if (!(retryAnalyzer instanceof RetryAnalyzer)) { + log().info("Using a non-default retry analyzer: {} on {}", retryAnalyzer.getClass().getSimpleName(), method.getName()); } } } diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/TesterraListener.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/TesterraListener.java index a80e223fab..c2b9934295 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/TesterraListener.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/TesterraListener.java @@ -42,6 +42,7 @@ import eu.tsystems.mms.tic.testframework.report.model.steps.TestStep; import eu.tsystems.mms.tic.testframework.report.utils.ExecutionContextController; import eu.tsystems.mms.tic.testframework.report.utils.IExecutionContextController; +import eu.tsystems.mms.tic.testframework.utils.Formatter; import org.apache.logging.log4j.core.LoggerContext; import org.testng.IConfigurable; import org.testng.IConfigurationListener; @@ -248,7 +249,8 @@ private MethodContext pBeforeInvocation( ITestResult testResult, ITestContext testContext ) { - final String methodName = getMethodName(testResult); + Formatter formatter = Testerra.getInjector().getInstance(Formatter.class); + final String methodName = formatter.getMethodName(testResult); // stores the actual testresult, auto-creates the method context MethodContext methodContext = ExecutionContextController.setCurrentMethodContext(testResult); @@ -320,8 +322,8 @@ private void pAfterInvocation( ITestResult testResult, ITestContext testContext ) { - - final String methodName = getMethodName(testResult); + Formatter formatter = Testerra.getInjector().getInstance(Formatter.class); + final String methodName = formatter.getMethodName(testResult); IExecutionContextController instance = Testerra.getInjector().getInstance(IExecutionContextController.class); // Optional optionalMethodContext = Optional.of(ExecutionContextController.getMethodContextFromTestResult(testResult)); @@ -330,11 +332,16 @@ private void pAfterInvocation( if (optionalMethodContext.isEmpty()) { if (testResult.getStatus() == ITestResult.CREATED || testResult.getStatus() == ITestResult.SKIP) { - /* - * TestNG bug or whatever ?!?! - */ + // Safely handling of skipped test methods ClassContext classContext = ExecutionContextController.getClassContextFromTestResult(testResult); - methodContext = classContext.safeAddSkipMethod(testResult); + methodContext = classContext.getMethodContext(testResult); + if (testResult.getThrowable() != null) { + methodContext.addError(testResult.getThrowable()); + } else { + methodContext.addError(new SkipException("Skipped")); + } + methodContext.setStatus(Status.SKIPPED); +// methodContext = classContext.safeAddSkipMethod(testResult); } else { log().error("INTERNAL ERROR. Could not create methodContext for {} with result {}", methodName, testResult); throw new SystemException("INTERNAL ERROR. Could not create methodContext for " + methodName + " with result: " + testResult); @@ -398,11 +405,15 @@ public void onTestSuccess(ITestResult iTestResult) { } /** - * This is only a fallback when 'afterInvocation was not called.' This could happen when TestNG runs into an exception, e.g. - * the Test method points to a non-existing dataprovider. + * This is method is only used in two cases: + * 1) Fallback method when 'afterInvocation' was not called. + * 2) When a data provider failed (only TestNG runner in IntelliJ). + *

+ * For 1) This could happen when TestNG runs into an exception, e.g. the Test method points to a non-existing dataprovider. * Therefor the events will only post if the corresponding MethodContext has no status. *

- * This method is also called when data provider failed. This is excluded, see {@link #onDataProviderFailure(ITestNGMethod, ITestContext, RuntimeException)} + * For 2) TestNG calls this method in that case, but the correct status 'SKIP' is set in + * {@link #onDataProviderFailure(ITestNGMethod, ITestContext, RuntimeException)} */ @Override public void onTestFailure(ITestResult iTestResult) { @@ -410,14 +421,16 @@ public void onTestFailure(ITestResult iTestResult) { return; } - InvokedMethod invokedMethod = new InvokedMethod(new Date().getTime(), iTestResult); - MethodContext methodContext = ExecutionContextController.getMethodContextFromTestResult(iTestResult); + Formatter formatter = Testerra.getInjector().getInstance(Formatter.class); + final String methodName = formatter.getMethodName(iTestResult); + MethodContext methodContext = ExecutionContextController.getMethodContextFromTestResult(iTestResult); + InvokedMethod invokedMethod = new InvokedMethod(new Date().getTime(), iTestResult); if (methodContext != null && methodContext.getStatus() == Status.NO_RUN) { AbstractMethodEvent event = new MethodEndEvent() .setTestResult(iTestResult) .setInvokedMethod(invokedMethod) - .setMethodName(getMethodName(iTestResult)) + .setMethodName(methodName) .setTestContext(iTestResult.getTestContext()) .setMethodContext(methodContext); Testerra.getEventBus().post(event); @@ -426,9 +439,10 @@ public void onTestFailure(ITestResult iTestResult) { } /** - * This method gets not only called when - * - a test was skipped using {@link Test#dependsOnMethods()} or by throwing a {@link SkipException}, - * - a data provider failed + * This method is called when + * - a test was skipped using {@link Test#dependsOnMethods()} or by throwing a {@link SkipException} + * - a configuration method failed + * - a data provider failed (non-IntelliJ execution) * - a failed test should not be retried by {@link RetryAnalyzer#retry(ITestResult)}. */ @Override @@ -509,12 +523,6 @@ public void beforeDataProviderExecution(IDataProviderMethod dataProviderMethod, } dataProviderSemaphore.put(testNGMethod, true); - // Creates a method context for test method in case of empty data provider - // Should solved by TestNG -// TestResult testMethodTestResult = TestResult.newContextAwareTestResult(testNGMethod, testContext); -// InvokedMethod testMethodInvokedMethod = new InvokedMethod(new Date().getTime(), testMethodTestResult); -// pBeforeInvocation(testMethodInvokedMethod, testMethodTestResult, testContext); - // Manually create a TestNG ConfigurationMethod and set the 'BeforeMethod' flavour IAnnotationFinder annoFinder = new DataProvAnnotationFinder(new DefaultAnnotationTransformer()); IObject.IdentifiableObject identifiableObject = new IObject.IdentifiableObject(dataProviderMethod.getInstance(), UUID.randomUUID()); @@ -569,18 +577,13 @@ public void onDataProviderFailure(ITestNGMethod testNGMethod, ITestContext testC dpTestResult.setThrowable(exception); pAfterInvocation(dpIinvokedMethod, dpTestResult, testContext); - // For the called test method a new method context is created + // For the called test method a new method context is created, the origin throwable from DP is appended ITestResult testResult = TestResult.newContextAwareTestResult(testNGMethod, testContext); - IInvokedMethod invokedMethod = new InvokedMethod(new Date().getTime(), testResult); - MethodContext methodContext = ExecutionContextController.getMethodContextFromTestResult(testResult); - Throwable nestedThrowable = exception.getCause() != null ? exception.getCause() : exception; - methodContext.addError(new SkipException("Method skipped because of failed data provider", nestedThrowable)); - - // Data provider methods are a kind of setup methods. If they crash the test method will get the status SKIPPED - methodContext.setStatus(Status.SKIPPED); + testResult.setThrowable(new SkipException("Method skipped because of failed data provider", nestedThrowable)); testResult.setStatus(ITestResult.SKIP); - pAfterInvocation(invokedMethod, testResult, testContext); + // invokedMethod is not needed/used + pAfterInvocation(null, testResult, testContext); // To prevent double method context, this map is needed dataProviderSemaphore.put(testNGMethod, true); diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/model/context/ClassContext.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/model/context/ClassContext.java index 3d0079df41..0e3722dc3d 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/report/model/context/ClassContext.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/report/model/context/ClassContext.java @@ -174,15 +174,15 @@ private MethodContext getMethodContext( return methodContext; } - public MethodContext safeAddSkipMethod(ITestResult testResult) { - MethodContext methodContext = getMethodContext(testResult); - methodContext.readErrors().findFirst().ifPresentOrElse( - error -> {}, - () -> { - methodContext.addError(new SkipException("Skipped")); - methodContext.setStatus(Status.SKIPPED); - } - ); - return methodContext; - } +// public MethodContext safeAddSkipMethod(ITestResult testResult) { +// MethodContext methodContext = getMethodContext(testResult); +// methodContext.readErrors().findFirst().ifPresentOrElse( +// error -> {}, +// () -> { +// methodContext.addError(new SkipException("Skipped")); +// methodContext.setStatus(Status.SKIPPED); +// } +// ); +// return methodContext; +// } } diff --git a/core/src/main/java/eu/tsystems/mms/tic/testframework/utils/Formatter.java b/core/src/main/java/eu/tsystems/mms/tic/testframework/utils/Formatter.java index dfa98c1716..c313321147 100644 --- a/core/src/main/java/eu/tsystems/mms/tic/testframework/utils/Formatter.java +++ b/core/src/main/java/eu/tsystems/mms/tic/testframework/utils/Formatter.java @@ -21,7 +21,9 @@ package eu.tsystems.mms.tic.testframework.utils; import org.testng.ITestNGMethod; +import org.testng.ITestResult; +import java.util.Arrays; import java.util.Date; /** @@ -50,12 +52,30 @@ default String DATE_TIME_FORMAT() { String logTime(Date date); - default String toString(ITestNGMethod method) { - return (method.isTest() ? "Test" : "Configuration") - .concat(" (") - .concat(method.getTestClass().getRealClass().getSimpleName()) + default String getMethodName(ITestResult testResult) { + ITestNGMethod testMethod = testResult.getMethod(); + Object[] parameters = testResult.getParameters(); + String parameterString = ""; + if (parameters != null) { + parameterString = Arrays.toString(parameters) + .replace("[", "") + .replace("]", ""); + } + + return testMethod.getTestClass().getRealClass().getSimpleName() .concat(".") - .concat(method.getMethodName()) - .concat("())"); + .concat(testMethod.getMethodName()) + .concat("(") + .concat(parameterString) + .concat(")"); + } + + default String toString(ITestResult testResult) { + ITestNGMethod testMethod = testResult.getMethod(); + + return (testMethod.isTest() ? "Test" : "Configuration") + .concat(" ") + .concat(this.getMethodName(testResult)); } + } diff --git a/integration-tests/src/main/java/eu/tsystems/mms/tic/testframework/core/utils/Log4jFileReader.java b/integration-tests/src/main/java/eu/tsystems/mms/tic/testframework/core/utils/Log4jFileReader.java index 35c9b606ba..18c7661bae 100644 --- a/integration-tests/src/main/java/eu/tsystems/mms/tic/testframework/core/utils/Log4jFileReader.java +++ b/integration-tests/src/main/java/eu/tsystems/mms/tic/testframework/core/utils/Log4jFileReader.java @@ -71,7 +71,9 @@ public List filterLogForTestMethod(final String classNameSlug, final Str } if (methodNameSlug != null) { - final String regEx = methodNameSlug.concat("\\b"); + final String regEx = methodNameSlug + .replace("(", "\\(") + .replace(")", "\\)"); final Pattern pattern = Pattern.compile(regEx); final Predicate predicate = s -> pattern.matcher(s).find(); diff --git a/integration-tests/src/test/java/io/testerra/test/pretest_status/DataProvider2Test.java b/integration-tests/src/test/java/io/testerra/test/pretest_status/DataProvider2Test.java new file mode 100644 index 0000000000..5399c7f779 --- /dev/null +++ b/integration-tests/src/test/java/io/testerra/test/pretest_status/DataProvider2Test.java @@ -0,0 +1,51 @@ +/* + * Testerra + * + * (C) 2024, Martin Großmann, Deutsche Telekom MMS GmbH, Deutsche Telekom AG + * + * Deutsche Telekom AG and all other contributors / + * copyright owners license this file to you under the Apache + * License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.testerra.test.pretest_status; + +import eu.tsystems.mms.tic.testframework.testing.TesterraTest; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Created on 2024-11-05 + * + * @author mgn + */ +public class DataProvider2Test extends TesterraTest { + + @BeforeMethod + public void beforeMethodOfDataProvider2Tests() { + Assert.fail("Before method has an assertion error."); + } + + @DataProvider + public Object[][] dataProviderOfDataProvider2Tests() { + return new String[][]{{"run01"}, {"run02"}}; + } + + @Test(dataProvider = "dataProviderOfDataProvider2Tests") + public void testT01_DataProviderWithFailedSetup(String dp) { + } + +} diff --git a/integration-tests/src/test/java/io/testerra/test/pretest_status/FailedSetupMethodTests.java b/integration-tests/src/test/java/io/testerra/test/pretest_status/FailedSetupMethodTests.java index 8591609746..3727630a3b 100644 --- a/integration-tests/src/test/java/io/testerra/test/pretest_status/FailedSetupMethodTests.java +++ b/integration-tests/src/test/java/io/testerra/test/pretest_status/FailedSetupMethodTests.java @@ -153,41 +153,4 @@ public void afterClassSetup02() { } } -// public static class SetupAfterTest implements Loggable { -// -// @Test -// public void testPassedMethodBeforeAFailedAfterMethod() { -// log().info("Info log of test method."); -// } -// -// @AfterMethod -// public void setupAfterMethod() { -// ASSERT.fail("Setup method must fail."); -// } -// } -// -// public static class SetupAfterClass1 implements Loggable { -// @Test -// public void testPassedMethodBeforeAFailedAfterClass() { -// log().info("Info log of test method."); -// } -// -// @AfterClass -// public void setupAfterClass() { -// ASSERT.fail("Setup method must fail."); -// } -// } -// -// public static class SetupAfterClass2 implements Loggable { -// @Test -// public void testPassedMethodBeforeAFailedAfterClass() { -// log().info("Info log of test method."); -// } -// -// @AfterClass -// public void setupAfterClass() { -// throw new RuntimeException("Setup method must fail."); -// } -// } - } diff --git a/integration-tests/src/test/java/io/testerra/test/status/CheckTestStatusTest.java b/integration-tests/src/test/java/io/testerra/test/status/CheckTestStatusTest.java index 6e38eb0336..8f5171c4e4 100644 --- a/integration-tests/src/test/java/io/testerra/test/status/CheckTestStatusTest.java +++ b/integration-tests/src/test/java/io/testerra/test/status/CheckTestStatusTest.java @@ -54,11 +54,12 @@ public static Object[][] provideExpectedStatusPerMethod() { {"test_invalidExpectedFailed_withMethod", Status.FAILED}, {"test_validExpectedFailed_withClass", Status.FAILED_EXPECTED}, {"test_invalidExpectedFailed_withClass", Status.FAILED}, - {"testT04_DataProviderWithFailedTests", Status.FAILED}, - {"testT04_DataProviderWithFailedTests", Status.PASSED}, - {"testT05_DataProviderWithFailedTestsOptional", Status.PASSED}, - {"testT06_DataProviderWithCollectedAssertions", Status.PASSED}, - {"testT06_DataProviderWithCollectedAssertions", Status.FAILED}, + {"testT04_DataProviderWithFailedTests(failed)", Status.FAILED}, + {"testT04_DataProviderWithFailedTests(passed)", Status.PASSED}, + {"testT05_DataProviderWithFailedTestsOptional(passed)", Status.PASSED}, + {"testT05_DataProviderWithFailedTestsOptional(failed)", Status.PASSED}, + {"testT06_DataProviderWithCollectedAssertions(passed)", Status.PASSED}, + {"testT06_DataProviderWithCollectedAssertions(failed)", Status.FAILED}, {"testT07_NonExistingDataProvider", Status.FAILED}, {"testT07_AfterClassWithAssertion", Status.PASSED}, {"testT08_AfterClassWithException", Status.PASSED}, @@ -78,25 +79,29 @@ public static Object[][] provideExpectedStatusPerMethod() { {"dataProviderThrowingAssertion", Status.FAILED}, {"dataProviderThrowingException", Status.FAILED}, {"dataProviderInClassThrowingException", Status.FAILED}, - {"beforeClassBeforeMethodSetup03", Status.NO_RUN}, - {"dataProviderWithNoData", Status.PASSED} + {"dataProviderWithNoData", Status.PASSED}, + {"beforeMethodOfDataProvider2Tests", Status.FAILED}, + {"dataProviderOfDataProvider2Tests", Status.PASSED} }; } @DataProvider public static Object[][] provideTestMethodsSkipped() { + final String failedDpMessage = "Method skipped because of failed data provider"; return new Object[][]{ - {"test_SkippedNoStatus", Status.NO_RUN, Status.SKIPPED.title}, - {"test_Skipped_dependingOnFailed", Status.NO_RUN, "depends on not successfully finished methods"}, - {"testT01_interceptCrashedDataProvider", Status.SKIPPED, "java.lang.AssertionError"}, - {"testT02_crashedDataProvider", Status.SKIPPED, "java.lang.AssertionError"}, - {"testT03_AssertFailedDataProvider", Status.SKIPPED, "java.lang.AssertionError"}, - {"testT01_BeforeClassWithAssertion", Status.NO_RUN, "java.lang.AssertionError"}, - {"testT02_BeforeClassWithException", Status.NO_RUN, "java.lang.RuntimeException"}, -// {"beforeClassBeforeMethodSetup03", Status.NO_RUN, ""}, // Cannot test here, because skipped setup method has no exception - {"testT03_BeforeMethodWithAssertion", Status.NO_RUN, "java.lang.AssertionError"}, - {"testT03_BeforeClassWithException", Status.NO_RUN, "java.lang.RuntimeException"}, - {"testT04_BeforeMethodWithException", Status.NO_RUN, "java.lang.RuntimeException"} + {"test_SkippedNoStatus", Status.SKIPPED, "org.testng.SkipException"}, + {"test_Skipped_dependingOnFailed", Status.SKIPPED, "depends on not successfully finished methods"}, + {"testT01_interceptCrashedDataProvider", Status.SKIPPED, failedDpMessage}, + {"testT02_crashedDataProvider", Status.SKIPPED, failedDpMessage}, + {"testT03_AssertFailedDataProvider", Status.SKIPPED, failedDpMessage}, + {"testT01_BeforeClassWithAssertion", Status.SKIPPED, "java.lang.AssertionError"}, + {"testT02_BeforeClassWithException", Status.SKIPPED, "java.lang.RuntimeException"}, + {"beforeClassBeforeMethodSetup03", Status.SKIPPED, ""}, + {"testT03_BeforeMethodWithAssertion", Status.SKIPPED, "java.lang.AssertionError"}, + {"testT03_BeforeClassWithException", Status.SKIPPED, "java.lang.RuntimeException"}, + {"testT04_BeforeMethodWithException", Status.SKIPPED, "java.lang.RuntimeException"}, + {"testT01_DataProviderWithFailedSetup(run01)", Status.SKIPPED, "java.lang.AssertionError"}, + {"testT01_DataProviderWithFailedSetup(run02)", Status.SKIPPED, "java.lang.AssertionError"} }; } @@ -127,13 +132,13 @@ public static Object[][] provideFinalTestResult() { return new Object[][]{ {"*** Stats: SuiteContexts: 3", "SuiteContext"}, {"*** Stats: TestContexts: 3", "TestContext"}, - {"*** Stats: ClassContexts: 16", "ClassContext"}, - {"*** Stats: MethodContexts: 84", "MethodContexts"}, - {"*** Stats: Test Methods Count: 60 (49 relevant)", "Test methods"}, + {"*** Stats: ClassContexts: 17", "ClassContext"}, + {"*** Stats: MethodContexts: 89", "MethodContexts"}, + {"*** Stats: Test Methods Count: 62 (51 relevant)", "Test methods"}, {"*** Stats: Failed: 10", "Failed tests"}, {"*** Stats: Retried: 11", "Retried tests"}, {"*** Stats: Expected Failed: 7", "Expected failed tests"}, - {"*** Stats: Skipped: 10", "Skipped tests"}, + {"*** Stats: Skipped: 12", "Skipped tests"}, {"*** Stats: Passed: 22 ⊃ Recovered: 5 ⊃ Repaired: 1", "Passed tests"} }; }