Merge pull request #461 from telekom/fix/method-context

Fix/method context

core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodContextUpdateWorker.java

@@ -58,19 +58,6 @@ public void onMethodEnd(MethodEndEvent event) {
                 testResult.setStatus(ITestResult.FAILURE);
                 methodContext.setStatus(Status.FAILED);
             }
-            StringBuilder sb = new StringBuilder();
-            sb.append("The following assertions failed in dataprovider method ");
-            sb.append(testResult.getMethod().getDataProviderMethod().getMethod().getName());
-            sb.append(":");
-            AtomicInteger i = new AtomicInteger();
-            methodContext.readErrors()
-//                    .filter(ErrorContext::isNotOptional)
-                    .forEach(errorContext -> {
-                        i.incrementAndGet();
-                        sb.append("\n").append(i).append(") ").append(errorContext.getThrowable().getMessage());
-                    });
-            AssertionError testMethodContainerError = new AssertionError(sb.toString());
-            testResult.setThrowable(testMethodContainerError);
         } else
             // Handle collected assertions if we have more than one
             if (testResult.isSuccess() && methodContext.readErrors().anyMatch(ErrorContext::isNotOptional)) {
@@ -128,6 +115,9 @@ public void onMethodEnd(MethodEndEvent event) {
             });
             TestStep failedStep = methodContext.getCurrentTestStep();
             methodContext.setFailedStep(failedStep);
+        } else if (event.isSkipped()) {
+            // Can be caused by failed dataprovider or config methods
+            methodContext.setStatus(Status.SKIPPED);
         } else if (testResult.isSuccess()) {
             methodContext.setStatus(Status.PASSED);
             RetryAnalyzer.methodHasBeenPassed(methodContext);

core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/finish/MethodEndWorker.java

@@ -45,7 +45,7 @@ public void onMethodEnd(MethodEndEvent event) {
         ITestNGMethod testMethod = event.getTestMethod();
         MethodContext methodContext = event.getMethodContext();
 
-        String msg = String.format("%s %s", methodContext.getStatus().title, formatter.toString(testMethod));
+        String msg = String.format("%s %s", methodContext.getStatus().title, formatter.toString(testResult));
         if (event.isFailed()) {
             log().error(msg, testResult.getThrowable());
         } else if (event.getTestResult().isSuccess()) {

core/src/main/java/eu/tsystems/mms/tic/testframework/execution/testng/worker/start/MethodStartWorker.java

@@ -22,18 +22,17 @@
 package eu.tsystems.mms.tic.testframework.execution.testng.worker.start;
 
 import com.google.common.eventbus.Subscribe;
-import eu.tsystems.mms.tic.testframework.annotations.NoRetry;
 import eu.tsystems.mms.tic.testframework.common.Testerra;
 import eu.tsystems.mms.tic.testframework.events.MethodStartEvent;
 import eu.tsystems.mms.tic.testframework.execution.testng.RetryAnalyzer;
 import eu.tsystems.mms.tic.testframework.logging.Loggable;
-import eu.tsystems.mms.tic.testframework.utils.DefaultFormatter;
 import eu.tsystems.mms.tic.testframework.utils.Formatter;
-import java.lang.reflect.Method;
 import org.testng.IRetryAnalyzer;
 import org.testng.ITestNGMethod;
 import org.testng.internal.annotations.DisabledRetryAnalyzer;
 
+import java.lang.reflect.Method;
+
 public class MethodStartWorker implements Loggable, MethodStartEvent.Listener {
 
     private final Formatter formatter = Testerra.getInjector().getInstance(Formatter.class);
@@ -46,7 +45,7 @@ public void onMethodStart(MethodStartEvent event) {
             addRetryAnalyzer(event);
         }
 
-        log().info("Run " + formatter.toString(event.getTestMethod()));
+        log().info("Run {}", formatter.toString(event.getTestResult()));
     }
 
     private void addRetryAnalyzer(MethodStartEvent event) {
@@ -55,8 +54,8 @@ private void addRetryAnalyzer(MethodStartEvent event) {
         Method method = event.getMethod();
         if (retryAnalyzer == null || retryAnalyzer instanceof DisabledRetryAnalyzer) {
             testNGMethod.setRetryAnalyzerClass(RetryAnalyzer.class);
-        } else if (!(retryAnalyzer instanceof RetryAnalyzer)){
-            log().info("Using a non-default retry analyzer: " + retryAnalyzer.getClass().getSimpleName() + " on " + method.getName());
+        } else if (!(retryAnalyzer instanceof RetryAnalyzer)) {
+            log().info("Using a non-default retry analyzer: {} on {}", retryAnalyzer.getClass().getSimpleName(), method.getName());
         }
     }
 }

core/src/main/java/eu/tsystems/mms/tic/testframework/report/TesterraListener.java

@@ -42,6 +42,7 @@
 import eu.tsystems.mms.tic.testframework.report.model.steps.TestStep;
 import eu.tsystems.mms.tic.testframework.report.utils.ExecutionContextController;
 import eu.tsystems.mms.tic.testframework.report.utils.IExecutionContextController;
+import eu.tsystems.mms.tic.testframework.utils.Formatter;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.testng.IConfigurable;
 import org.testng.IConfigurationListener;
@@ -248,7 +249,8 @@ private MethodContext pBeforeInvocation(
             ITestResult testResult,
             ITestContext testContext
     ) {
-        final String methodName = getMethodName(testResult);
+        Formatter formatter = Testerra.getInjector().getInstance(Formatter.class);
+        final String methodName = formatter.getMethodName(testResult);
 
         // stores the actual testresult, auto-creates the method context
         MethodContext methodContext = ExecutionContextController.setCurrentMethodContext(testResult);
@@ -291,22 +293,22 @@ public void afterInvocation(
         pAfterInvocation(method, testResult, context);
     }
 
-    private static String getMethodName(ITestResult testResult) {
-        ITestNGMethod testMethod = testResult.getMethod();
-        String methodName = testMethod.getMethodName();
-        Object[] parameters = testResult.getParameters();
-        if (parameters != null) {
-            methodName += "(";
-            for (Object parameter : parameters) {
-                methodName += parameter + ", ";
-            }
-            if (parameters.length > 0) {
-                methodName = methodName.substring(0, methodName.length() - 2);
-            }
-            methodName += ")";
-        }
-        return methodName;
-    }
+//    private static String getMethodName(ITestResult testResult) {
+//        ITestNGMethod testMethod = testResult.getMethod();
+//        String methodName = testMethod.getMethodName();
+//        Object[] parameters = testResult.getParameters();
+//        if (parameters != null) {
+//            methodName += "(";
+//            for (Object parameter : parameters) {
+//                methodName += parameter + ", ";
+//            }
+//            if (parameters.length > 0) {
+//                methodName = methodName.substring(0, methodName.length() - 2);
+//            }
+//            methodName += ")";
+//        }
+//        return methodName;
+//    }
 
     /**
      * Override after invocation, to visualize threads and finish reporting.
@@ -320,8 +322,8 @@ private void pAfterInvocation(
             ITestResult testResult,
             ITestContext testContext
     ) {
-
-        final String methodName = getMethodName(testResult);
+        Formatter formatter = Testerra.getInjector().getInstance(Formatter.class);
+        final String methodName = formatter.getMethodName(testResult);
         IExecutionContextController instance = Testerra.getInjector().getInstance(IExecutionContextController.class);
 
 //        Optional<MethodContext> optionalMethodContext = Optional.of(ExecutionContextController.getMethodContextFromTestResult(testResult));
@@ -330,11 +332,16 @@ private void pAfterInvocation(
 
         if (optionalMethodContext.isEmpty()) {
             if (testResult.getStatus() == ITestResult.CREATED || testResult.getStatus() == ITestResult.SKIP) {
-                /*
-                 * TestNG bug or whatever ?!?!
-                 */
+                // Safely handling of skipped test methods
                 ClassContext classContext = ExecutionContextController.getClassContextFromTestResult(testResult);
-                methodContext = classContext.safeAddSkipMethod(testResult);
+                methodContext = classContext.getMethodContext(testResult);
+                if (testResult.getThrowable() != null) {
+                    methodContext.addError(testResult.getThrowable());
+                } else {
+                    methodContext.addError(new SkipException("Skipped"));
+                }
+                methodContext.setStatus(Status.SKIPPED);
+//                methodContext = classContext.safeAddSkipMethod(testResult);
             } else {
                 log().error("INTERNAL ERROR. Could not create methodContext for {} with result {}", methodName, testResult);
                 throw new SystemException("INTERNAL ERROR. Could not create methodContext for " + methodName + " with result: " + testResult);
@@ -398,26 +405,32 @@ public void onTestSuccess(ITestResult iTestResult) {
     }
 
     /**
-     * This is only a fallback when 'afterInvocation was not called.' This could happen when TestNG runs into an exception, e.g.
-     * the Test method points to a non-existing dataprovider.
+     * This is method is only used in two cases:
+     * 1) Fallback method when 'afterInvocation' was not called.
+     * 2) When a data provider failed (only TestNG runner in IntelliJ).
+     * <p>
+     * For 1) This could happen when TestNG runs into an exception, e.g. the Test method points to a non-existing dataprovider.
      * Therefor the events will only post if the corresponding MethodContext has no status.
      * <p>
-     * This method is also called when data provider failed. This is excluded, see {@link #onDataProviderFailure(ITestNGMethod, ITestContext, RuntimeException)}
+     * For 2) TestNG calls this method in that case, but the correct status 'SKIP' is set in
+     * {@link #onDataProviderFailure(ITestNGMethod, ITestContext, RuntimeException)}
      */
     @Override
     public void onTestFailure(ITestResult iTestResult) {
         if (dataProviderSemaphore.containsKey(iTestResult.getMethod())) {
             return;
         }
 
-        InvokedMethod invokedMethod = new InvokedMethod(new Date().getTime(), iTestResult);
-        MethodContext methodContext = ExecutionContextController.getMethodContextFromTestResult(iTestResult);
+        Formatter formatter = Testerra.getInjector().getInstance(Formatter.class);
+        final String methodName = formatter.getMethodName(iTestResult);
 
+        MethodContext methodContext = ExecutionContextController.getMethodContextFromTestResult(iTestResult);
+        InvokedMethod invokedMethod = new InvokedMethod(new Date().getTime(), iTestResult);
         if (methodContext != null && methodContext.getStatus() == Status.NO_RUN) {
             AbstractMethodEvent event = new MethodEndEvent()
                     .setTestResult(iTestResult)
                     .setInvokedMethod(invokedMethod)
-                    .setMethodName(getMethodName(iTestResult))
+                    .setMethodName(methodName)
                     .setTestContext(iTestResult.getTestContext())
                     .setMethodContext(methodContext);
             Testerra.getEventBus().post(event);
@@ -426,9 +439,10 @@ public void onTestFailure(ITestResult iTestResult) {
     }
 
     /**
-     * This method gets not only called when
-     * - a test was skipped using {@link Test#dependsOnMethods()} or by throwing a {@link SkipException},
-     * - a data provider failed
+     * This method is called when
+     * - a test was skipped using {@link Test#dependsOnMethods()} or by throwing a {@link SkipException}
+     * - a configuration method failed
+     * - a data provider failed (non-IntelliJ execution)
      * - a failed test should not be retried by {@link RetryAnalyzer#retry(ITestResult)}.
      */
     @Override
@@ -509,12 +523,6 @@ public void beforeDataProviderExecution(IDataProviderMethod dataProviderMethod,
         }
         dataProviderSemaphore.put(testNGMethod, true);
 
-        // Creates a method context for test method in case of empty data provider
-        // Should solved by TestNG
-//        TestResult testMethodTestResult = TestResult.newContextAwareTestResult(testNGMethod, testContext);
-//        InvokedMethod testMethodInvokedMethod = new InvokedMethod(new Date().getTime(), testMethodTestResult);
-//        pBeforeInvocation(testMethodInvokedMethod, testMethodTestResult, testContext);
-
         // Manually create a TestNG ConfigurationMethod and set the 'BeforeMethod' flavour
         IAnnotationFinder annoFinder = new DataProvAnnotationFinder(new DefaultAnnotationTransformer());
         IObject.IdentifiableObject identifiableObject = new IObject.IdentifiableObject(dataProviderMethod.getInstance(), UUID.randomUUID());
@@ -569,18 +577,13 @@ public void onDataProviderFailure(ITestNGMethod testNGMethod, ITestContext testC
         dpTestResult.setThrowable(exception);
         pAfterInvocation(dpIinvokedMethod, dpTestResult, testContext);
 
-        // For the called test method a new method context is created
+        // For the called test method a new method context is created, the origin throwable from DP is appended
         ITestResult testResult = TestResult.newContextAwareTestResult(testNGMethod, testContext);
-        IInvokedMethod invokedMethod = new InvokedMethod(new Date().getTime(), testResult);
-        MethodContext methodContext = ExecutionContextController.getMethodContextFromTestResult(testResult);
-
         Throwable nestedThrowable = exception.getCause() != null ? exception.getCause() : exception;
-        methodContext.addError(new SkipException("Method skipped because of failed data provider", nestedThrowable));
-
-        // Data provider methods are a kind of setup methods. If they crash the test method will get the status SKIPPED
-        methodContext.setStatus(Status.SKIPPED);
+        testResult.setThrowable(new SkipException("Method skipped because of failed data provider", nestedThrowable));
         testResult.setStatus(ITestResult.SKIP);
-        pAfterInvocation(invokedMethod, testResult, testContext);
+        // invokedMethod is not needed/used
+        pAfterInvocation(null, testResult, testContext);
 
         // To prevent double method context, this map is needed
         dataProviderSemaphore.put(testNGMethod, true);

core/src/main/java/eu/tsystems/mms/tic/testframework/report/model/context/ClassContext.java

@@ -174,15 +174,15 @@ private MethodContext getMethodContext(
         return methodContext;
     }
 
-    public MethodContext safeAddSkipMethod(ITestResult testResult) {
-        MethodContext methodContext = getMethodContext(testResult);
-        methodContext.readErrors().findFirst().ifPresentOrElse(
-                error -> {},
-                () -> {
-                    methodContext.addError(new SkipException("Skipped"));
-                    methodContext.setStatus(Status.SKIPPED);
-                }
-        );
-        return methodContext;
-    }
+//    public MethodContext safeAddSkipMethod(ITestResult testResult) {
+//        MethodContext methodContext = getMethodContext(testResult);
+//        methodContext.readErrors().findFirst().ifPresentOrElse(
+//                error -> {},
+//                () -> {
+//                    methodContext.addError(new SkipException("Skipped"));
+//                    methodContext.setStatus(Status.SKIPPED);
+//                }
+//        );
+//        return methodContext;
+//    }
 }

core/src/main/java/eu/tsystems/mms/tic/testframework/utils/Formatter.java

@@ -21,7 +21,9 @@
 package eu.tsystems.mms.tic.testframework.utils;
 
 import org.testng.ITestNGMethod;
+import org.testng.ITestResult;
 
+import java.util.Arrays;
 import java.util.Date;
 
 /**
@@ -50,12 +52,30 @@ default String DATE_TIME_FORMAT() {
 
     String logTime(Date date);
 
-    default String toString(ITestNGMethod method) {
-        return (method.isTest() ? "Test" : "Configuration")
-                .concat(" (")
-                .concat(method.getTestClass().getRealClass().getSimpleName())
+    default String getMethodName(ITestResult testResult) {
+        ITestNGMethod testMethod = testResult.getMethod();
+        Object[] parameters = testResult.getParameters();
+        String parameterString = "";
+        if (parameters != null) {
+            parameterString = Arrays.toString(parameters)
+                    .replace("[", "")
+                    .replace("]", "");
+        }
+
+        return testMethod.getTestClass().getRealClass().getSimpleName()
                 .concat(".")
-                .concat(method.getMethodName())
-                .concat("())");
+                .concat(testMethod.getMethodName())
+                .concat("(")
+                .concat(parameterString)
+                .concat(")");
+    }
+
+    default String toString(ITestResult testResult) {
+        ITestNGMethod testMethod = testResult.getMethod();
+
+        return (testMethod.isTest() ? "Test" : "Configuration")
+                .concat(" ")
+                .concat(this.getMethodName(testResult));
     }
+
 }

integration-tests/src/main/java/eu/tsystems/mms/tic/testframework/core/utils/Log4jFileReader.java

@@ -71,7 +71,9 @@ public List<String> filterLogForTestMethod(final String classNameSlug, final Str
         }
 
         if (methodNameSlug != null) {
-            final String regEx = methodNameSlug.concat("\\b");
+            final String regEx = methodNameSlug
+                    .replace("(", "\\(")
+                    .replace(")", "\\)");
             final Pattern pattern = Pattern.compile(regEx);
             final Predicate<String> predicate = s -> pattern.matcher(s).find();
 

integration-tests/src/test/java/io/testerra/test/pretest_status/DataProvider2Test.java

@@ -0,0 +1,51 @@
+/*
+ * Testerra
+ *
+ * (C) 2024, Martin Großmann,  Deutsche Telekom MMS GmbH, Deutsche Telekom AG
+ *
+ * Deutsche Telekom AG and all other contributors /
+ * copyright owners license this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this
+ * file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.testerra.test.pretest_status;
+
+import eu.tsystems.mms.tic.testframework.testing.TesterraTest;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/**
+ * Created on 2024-11-05
+ *
+ * @author mgn
+ */
+public class DataProvider2Test extends TesterraTest {
+
+    @BeforeMethod
+    public void beforeMethodOfDataProvider2Tests() {
+        Assert.fail("Before method has an assertion error.");
+    }
+
+    @DataProvider
+    public Object[][] dataProviderOfDataProvider2Tests() {
+        return new String[][]{{"run01"}, {"run02"}};
+    }
+
+    @Test(dataProvider = "dataProviderOfDataProvider2Tests")
+    public void testT01_DataProviderWithFailedSetup(String dp) {
+    }
+
+}