diff --git a/src/main/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/synchronize/AzureDevOpsResultSynchronizer.java b/src/main/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/synchronize/AzureDevOpsResultSynchronizer.java index 7daa695..76ad10e 100644 --- a/src/main/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/synchronize/AzureDevOpsResultSynchronizer.java +++ b/src/main/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/synchronize/AzureDevOpsResultSynchronizer.java @@ -33,17 +33,16 @@ import eu.tsystems.mms.tic.testerra.plugins.azuredevops.mapper.RunState; import eu.tsystems.mms.tic.testerra.plugins.azuredevops.mapper.Testplan; import eu.tsystems.mms.tic.testerra.plugins.azuredevops.restclient.AzureDevOpsClient; -import eu.tsystems.mms.tic.testframework.annotations.Fails; -import eu.tsystems.mms.tic.testframework.connectors.util.AbstractCommonSynchronizer; -import eu.tsystems.mms.tic.testframework.events.MethodEndEvent; +import eu.tsystems.mms.tic.testframework.events.TestStatusUpdateEvent; import eu.tsystems.mms.tic.testframework.logging.Loggable; +import eu.tsystems.mms.tic.testframework.report.Status; import eu.tsystems.mms.tic.testframework.report.model.context.ErrorContext; +import eu.tsystems.mms.tic.testframework.report.model.context.MethodContext; import org.apache.commons.lang3.exception.ExceptionUtils; import java.lang.reflect.Method; import java.time.Instant; import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.Optional; @@ -52,7 +51,7 @@ * * @author mgn */ -public class AzureDevOpsResultSynchronizer extends AbstractCommonSynchronizer implements Loggable { +public class AzureDevOpsResultSynchronizer implements TestStatusUpdateEvent.Listener, Loggable { private AzureDevOpsClient client = null; @@ -65,25 +64,55 @@ public AzureDevOpsResultSynchronizer() { } @Override - protected void pOnTestSuccess(MethodEndEvent event) { - log().info("Method " + event.getTestMethod().getMethodName() + " passed"); - this.syncTestresult(event, Outcome.PASSED); - } + public void onTestStatusUpdate(TestStatusUpdateEvent event) { - @Override - protected void pOnTestFailure(MethodEndEvent event) { - log().info("Method " + event.getTestMethod().getMethodName() + " failed"); - // Only the last execution of a failed tests is synced - // Otherwise the run contains more results of the same test method. - this.syncTestresult(event, Outcome.FAILED); + MethodContext methodContext = event.getMethodContext(); + Status status = methodContext.getStatus(); + switch (status) { + case PASSED: + case REPAIRED: + case RECOVERED: + log().info("Method " + methodContext.getName() + " passed."); + this.syncTestresult(event, Outcome.PASSED); + break; + case FAILED: + case FAILED_EXPECTED: + log().info("Method " + methodContext.getName() + " failed."); + this.syncTestresult(event, Outcome.FAILED); + break; + case RETRIED: + log().info("Method " + methodContext.getName() + " was retried and will not sync."); + break; + case SKIPPED: + log().info("Method " + methodContext.getName() + " skipped."); + this.syncTestresult(event, Outcome.NOT_EXECUTED); + break; + default: + log().info(String.format("Method state %s of %s cannot handle.", status.toString(), methodContext.getName())); + } } - @Override - protected void pOnTestSkip(MethodEndEvent event) { - log().info("Method " + event.getTestMethod().getMethodName() + " skipped"); - this.syncTestresult(event, Outcome.NOT_EXECUTED); - } +// @Override +// protected void pOnTestSuccess(MethodEndEvent event) { +// log().info("Method " + event.getTestMethod().getMethodName() + " passed"); +// this.syncTestresult(event, Outcome.PASSED); +// } +// +// @Override +// protected void pOnTestFailure(MethodEndEvent event) { +// log().info("Method " + event.getTestMethod().getMethodName() + " failed"); +// // Only the last execution of a failed tests is synced +// // Otherwise the run contains more results of the same test method. +// this.syncTestresult(event, Outcome.FAILED); +// +// } +// +// @Override +// protected void pOnTestSkip(MethodEndEvent event) { +// log().info("Method " + event.getTestMethod().getMethodName() + " skipped"); +// this.syncTestresult(event, Outcome.NOT_EXECUTED); +// } private void init() { this.config = AzureDevOpsConfig.getInstance(); @@ -121,72 +150,79 @@ private void init() { * @param event * @param outcome */ - private synchronized void syncTestresult(MethodEndEvent event, Outcome outcome) { - AzureTest annotation = this.getAnnotation(event); - if (this.config.isAzureSyncEnabled() && annotation != null && annotation.enabled()) { - - Points points = this.client.getPointsByTestCaseFilter(annotation.id()); - if (points.getPoints().size() > 0) { - - // Find the point with the current test plan id - Optional optionalPoint = points.getPoints().stream().filter(point -> this.config.getAzureTestPlanId() == point.getTestPlan().getId()).findFirst(); - if (optionalPoint.isPresent()) { - - // Create a new test result based on current test method - Result result = new Result(); - result.setTestPoint(optionalPoint.get()); - if (!outcome.equals(Outcome.NOT_EXECUTED)) { - result.setStartedDate(event.getMethodContext().getStartTime().toInstant().toString()); - // Method context contains no end time at this moment - //result.setCompletedDate(event.getMethodContext().getEndTime().toInstant().toString()); - result.setCompletedDate(new Date().toInstant().toString()); - } - result.setOutcome(outcome.toString()); - // Priority is taken from test case, cannot change with test result -// result.setPriority(this.getPriorityByFailureCorridor(event.getMethodContext().failureCorridorValue)); - - if (outcome.equals(Outcome.FAILED)) { - event.getMethodContext() - .readErrors() - .filter(ErrorContext::isNotOptional) - .findFirst() - .ifPresent(errorContext -> { - Throwable throwable = errorContext.getThrowable(); - final String errorMessage = throwable.getMessage(); - result.setErrorMessage(errorMessage); - result.setFailureType(this.getFailureType(event).toString()); - final String stackTrace = ExceptionUtils.getStackTrace(throwable); - result.setStackTrace(stackTrace); - }); + private synchronized void syncTestresult(TestStatusUpdateEvent event, Outcome outcome) { + Optional method = this.getMethodFromEvent(event); + + if (method.isPresent()) { + + Optional annotation = this.getAnnotation(event); + if (this.config.isAzureSyncEnabled() && annotation.isPresent()) { + + if (annotation.get().enabled()) { + + Points points = this.client.getPointsByTestCaseFilter(annotation.get().id()); + if (points.getPoints().size() > 0) { + + // Find the point with the current test plan id + Optional optionalPoint = points.getPoints().stream().filter(point -> this.config.getAzureTestPlanId() == point.getTestPlan().getId()).findFirst(); + if (optionalPoint.isPresent()) { + + // Create a new test result based on current test method + Result result = new Result(); + result.setTestPoint(optionalPoint.get()); + if (!outcome.equals(Outcome.NOT_EXECUTED)) { + result.setStartedDate(event.getMethodContext().getStartTime().toInstant().toString()); + result.setCompletedDate(event.getMethodContext().getEndTime().toInstant().toString()); + } + result.setOutcome(outcome.toString()); + // Priority is taken from test case, cannot change with test result + // result.setPriority(this.getPriorityByFailureCorridor(event.getMethodContext().failureCorridorValue)); + + if (outcome.equals(Outcome.FAILED)) { + event.getMethodContext() + .readErrors() + .filter(ErrorContext::isNotOptional) + .findFirst() + .ifPresent(errorContext -> { + Throwable throwable = errorContext.getThrowable(); + final String errorMessage = throwable.getMessage(); + result.setErrorMessage(errorMessage); + result.setFailureType(this.getFailureType(event).toString()); + final String stackTrace = ExceptionUtils.getStackTrace(throwable); + result.setStackTrace(stackTrace); + }); + } + + List resultList = new ArrayList<>(); + resultList.add(result); + this.client.addResult(resultList, this.currentRunId); + + } else { + log().warn(String.format( + "Cannot sync %s: Testcase with ID %s is not mapped to test plan %s", + method.get().getName(), + annotation.get().id(), + this.config.getAzureTestPlanId() + )); + } + + } else { + log().warn(String.format( + "Cannot sync %s: Testcase with ID %s was not added to a test plan.", + method.get().getName(), + annotation.get().id())); } - List resultList = new ArrayList<>(); - resultList.add(result); - this.client.addResult(resultList, this.currentRunId); - - } else { - log().warn(String.format( - "Cannot sync %s: Testcase with ID %s is not mapped to test plan %s", - event.getTestMethod().getMethodName(), - annotation.id(), - this.config.getAzureTestPlanId() - )); + } // end method enabled sync + else { + log().info(String.format("Sync of %s is deactivated.", method.get().getName())); } - } else { - log().warn(String.format( - "Cannot sync %s: Testcase with ID %s was not added to a test plan.", - event.getTestMethod().getMethodName(), - annotation.id())); - } + } // end active sync and annotation found + + } // end method.isPresent + - } - if (annotation != null && !annotation.enabled()) { - log().info(String.format( - "Sync of %s is deactivated.", - event.getTestMethod().getMethodName() - )); - } } public void shutdown() { @@ -206,23 +242,37 @@ public void shutdown() { } } - private AzureTest getAnnotation(MethodEndEvent event) { - final Method method = event.getTestResult().getMethod().getConstructorOrMethod().getMethod(); - if (method.isAnnotationPresent(AzureTest.class)) { - return method.getAnnotation(AzureTest.class); + private Optional getAnnotation(TestStatusUpdateEvent event) { + Optional methodFromEvent = this.getMethodFromEvent(event); + if (methodFromEvent.isPresent() && methodFromEvent.get().isAnnotationPresent(AzureTest.class)) { + return Optional.of(methodFromEvent.get().getAnnotation(AzureTest.class)); } else { log().info("No annoation found for sync results with Azure DevOps"); - return null; + return Optional.empty(); } } - private FailureType getFailureType(MethodEndEvent event) { - final Method method = event.getTestResult().getMethod().getConstructorOrMethod().getMethod(); - if (method.isAnnotationPresent(Fails.class)) { + private Optional getMethodFromEvent(TestStatusUpdateEvent event) { + return event.getMethodContext().getTestNgResult() + .map(iTestResult -> iTestResult.getMethod().getConstructorOrMethod().getMethod()); + } + + private FailureType getFailureType(TestStatusUpdateEvent event) { + if (event.getMethodContext().getStatus() == Status.FAILED_EXPECTED) { return FailureType.KNOWN_ISSUE; } else { return FailureType.NEW_ISSUE; } } +// private FailureType getFailureType(MethodEndEvent event) { +// final Method method = event.getTestResult().getMethod().getConstructorOrMethod().getMethod(); +// if (method.isAnnotationPresent(Fails.class)) { +// return FailureType.KNOWN_ISSUE; +// } else { +// return FailureType.NEW_ISSUE; +// } +// } + + } diff --git a/src/test/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/tests/PlaygroundTests.java b/src/test/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/tests/PlaygroundTests.java index afef2f8..751549c 100644 --- a/src/test/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/tests/PlaygroundTests.java +++ b/src/test/java/eu/tsystems/mms/tic/testerra/plugins/azuredevops/tests/PlaygroundTests.java @@ -24,6 +24,7 @@ import eu.tsystems.mms.tic.testerra.plugins.azuredevops.annotation.AzureTest; import eu.tsystems.mms.tic.testframework.annotations.Fails; +import eu.tsystems.mms.tic.testframework.execution.testng.RetryAnalyzer; import eu.tsystems.mms.tic.testframework.testing.TesterraTest; import eu.tsystems.mms.tic.testframework.utils.TimerUtils; import org.testng.Assert; @@ -31,6 +32,7 @@ import org.testng.annotations.Test; import java.lang.reflect.Method; +import java.util.concurrent.atomic.AtomicInteger; /** * Created on 26.11.2020 @@ -61,4 +63,19 @@ public void test_Passed02() { Assert.assertTrue(false, "Mega exception"); } + AtomicInteger counter = new AtomicInteger(0); + + @AzureTest(id = 2284) + @Test(priority = 6) + public void test_03RetriedTest() { + this.counter.incrementAndGet(); + if (counter.get() == 1) { + // Message is already defined in test.properties + Assert.assertTrue(false, "retry_for_azuredevops_connector"); + } else { + Assert.assertTrue(true); + } + + } + } diff --git a/src/test/resources/test.properties b/src/test/resources/test.properties index 4f68128..5eefdec 100644 --- a/src/test/resources/test.properties +++ b/src/test/resources/test.properties @@ -1 +1,3 @@ tt.report.name=DemoAzureDevops + +tt.failed.tests.if.throwable.messages=retry_for_azuredevops_connector