Support Visual Studio test reports as source.

Allow for using Visual Studio test reports (.trx) as source for the metrics 'tests', 'test cases', and 'source up-to-dateness'.

Closes [#10009]

components/collector/.vulture_ignore_list.py

@@ -157,6 +157,7 @@
 SchemaVersion  # unused variable (src/source_collectors/trivy/security_warnings.py:49)
 Results  # unused variable (src/source_collectors/trivy/security_warnings.py:50)
 TrivyJSONSecurityWarnings  # unused class (src/source_collectors/trivy/security_warnings.py:56)
+VisualStudioTRXSourceUpToDateness  # unused class (src/source_collectors/visual_studio_trx/source_up_to_dateness.py:13)
 totalCount  # unused variable (tests/source_collectors/github/test_merge_requests.py:16)
 baseRefName  # unused variable (tests/source_collectors/github/test_merge_requests.py:24)
 createdAt  # unused variable (tests/source_collectors/github/test_merge_requests.py:27)

components/collector/src/metric_collectors/test_cases.py

@@ -42,12 +42,34 @@ class TestCases(MetricCollector):
         ("errored", "errored"): "errored",
     }
     # Mapping to uniformize the test results from different sources:
-    UNIFORMIZED_TEST_RESULTS: ClassVar[dict[str, TestResult]] = {"fail": "failed", "pass": "passed", "skip": "skipped"}
+    UNIFORMIZED_TEST_RESULTS: ClassVar[dict[str, TestResult]] = {
+        "aborted": "errored",
+        "completed": "passed",
+        "disconnected": "errored",
+        "error": "errored",
+        "fail": "failed",
+        "inconclusive": "skipped",
+        "inprogress": "untested",
+        "notexecuted": "skipped",
+        "notrunnable": "skipped",
+        "pass": "passed",
+        "passedbutrunaborted": "passed",
+        "pending": "untested",
+        "skip": "skipped",
+        "timeout": "errored",
+        "warning": "errored",
+    }
     # Regular expression to identify test case ids in test names and descriptions:
     TEST_CASE_KEY_RE = re.compile(r"\w+\-\d+")
     # The supported source types for test cases and test reports:
     TEST_CASE_SOURCE_TYPES: ClassVar[list[str]] = ["jira"]
-    TEST_REPORT_SOURCE_TYPES: ClassVar[list[str]] = ["jenkins_test_report", "junit", "robot_framework", "testng"]
+    TEST_REPORT_SOURCE_TYPES: ClassVar[list[str]] = [
+        "jenkins_test_report",
+        "junit",
+        "robot_framework",
+        "testng",
+        "visual_studio_trx",
+    ]
 
     async def collect(self) -> MetricMeasurement | None:
         """Override to add the test results from the test report(s) to the test cases."""
@@ -113,4 +135,5 @@ def source_type(self, source: SourceMeasurement) -> str:
     @classmethod
     def test_result(cls, entity: Entity) -> TestResult:
         """Return the (uniformized) test result of the entity."""
-        return cls.UNIFORMIZED_TEST_RESULTS.get(entity["test_result"], entity["test_result"])
+        test_result = entity["test_result"].lower()
+        return cls.UNIFORMIZED_TEST_RESULTS.get(test_result, test_result)

components/collector/src/source_collectors/__init__.py

@@ -156,3 +156,6 @@
 from .trello.issues import TrelloIssues
 from .trello.source_up_to_dateness import TrelloSourceUpToDateness
 from .trivy.security_warnings import TrivyJSONSecurityWarnings
+from .visual_studio_trx.source_up_to_dateness import VisualStudioTRXSourceUpToDateness
+from .visual_studio_trx.test_cases import VisualStudioTRXTestCases
+from .visual_studio_trx.tests import VisualStudioTRXTests

components/collector/src/source_collectors/visual_studio_trx/source_up_to_dateness.py

@@ -0,0 +1,22 @@
+"""Visual Studio TRX source up-to-dateness collector."""
+
+from datetime import datetime
+
+from shared.utils.date_time import now
+
+from base_collectors import TimePassedCollector, XMLFileSourceCollector
+from collector_utilities.date_time import parse_datetime
+from collector_utilities.functions import parse_source_response_xml_with_namespace
+from collector_utilities.type import Response
+
+
+class VisualStudioTRXSourceUpToDateness(XMLFileSourceCollector, TimePassedCollector):
+    """Collector to collect the Visual Studio TRX report age."""
+
+    async def _parse_source_response_date_time(self, response: Response) -> datetime:
+        """Override to parse the timestamp from the response."""
+        tree, namespaces = await parse_source_response_xml_with_namespace(response)
+        times = tree.find("./ns:Times", namespaces)
+        return (
+            now() if times is None else parse_datetime(times.attrib["creation"])
+        )  # The creation attribute is required

components/collector/src/source_collectors/visual_studio_trx/test_cases.py

@@ -0,0 +1,7 @@
+"""Visual Studio TRX test cases collector."""
+
+from .tests import VisualStudioTRXTests
+
+
+class VisualStudioTRXTestCases(VisualStudioTRXTests):
+    """Collector for Visual Studio TRX test cases."""

components/collector/src/source_collectors/visual_studio_trx/tests.py

@@ -0,0 +1,47 @@
+"""Visual Studio TRX tests collector."""
+
+import re
+from typing import cast
+from xml.etree.ElementTree import Element  # nosec # Element is not available from defusedxml, but only used as type
+
+from base_collectors import XMLFileSourceCollector
+from collector_utilities.functions import parse_source_response_xml_with_namespace
+from collector_utilities.type import Namespaces
+from metric_collectors.test_cases import TestCases
+from model import Entities, Entity, SourceMeasurement, SourceResponses
+
+
+class VisualStudioTRXTests(XMLFileSourceCollector):
+    """Collector for Visual Studio TRX tests."""
+
+    async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
+        """Override to parse the tests from the Visual Studio TRX report."""
+        entities = Entities()
+        test_results = {}
+        total = 0
+        for response in responses:
+            tree, namespaces = await parse_source_response_xml_with_namespace(response)
+            for result in tree.findall(".//ns:UnitTestResult", namespaces):
+                test_results[result.attrib["testId"]] = result.attrib["outcome"]
+            for test in tree.findall(".//ns:UnitTest", namespaces):
+                parsed_entity = self.__entity(test, test_results[test.attrib["id"]], namespaces)
+                if self._include_entity(parsed_entity):
+                    entities.append(parsed_entity)
+                total += 1
+        return SourceMeasurement(entities=entities, total=str(total))
+
+    def _include_entity(self, entity: Entity) -> bool:
+        """Return whether to include the entity in the measurement."""
+        test_results_to_count = cast(list[str], self._parameter("test_result"))
+        return entity["test_result"] in test_results_to_count
+
+    @staticmethod
+    def __entity(test: Element, result: str, namespaces: Namespaces) -> Entity:
+        """Transform a test case into a test entity."""
+        name = test.attrib["name"]
+        for category in test.findall(".//ns:TestCategoryItem", namespaces):
+            if match := re.search(TestCases.TEST_CASE_KEY_RE, category.attrib["TestCategory"]):
+                name += f" ({match[0]})"
+                break
+        key = test.attrib["id"]
+        return Entity(key=key, name=name, test_result=result)

components/collector/tests/source_collectors/visual_studio_trx/base.py

@@ -0,0 +1,70 @@
+"""Base classes for Visual Studio TRX test report collector unit tests."""
+
+from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase
+
+
+class VisualStudioTRXCollectorTestCase(SourceCollectorTestCase):
+    """Base class for Visual Studio TRX collector unit tests."""
+
+    SOURCE_TYPE = "visual_studio_trx"
+    VISUAL_STUDIO_TRX_XML = r"""<?xml version="1.0" encoding="utf-8"?>
+    <TestRun xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
+        <Times creation="2024-09-12T11:33:30.3272909+02:00" />
+        <Results>
+            <UnitTestResult
+                executionId="268fd488-12c7-4107-80d8-5df1200cd637"
+                testId="63eb0c90-d1fc-a21e-6fc0-3974b0cc65db"
+                testName="BestaandeZaakOpenen2"
+                computerName="XYZ-BLA-24"
+                duration="00:00:01.4635437"
+                startTime="2024-09-12T11:33:29.3938415+02:00"
+                endTime="2024-09-12T11:33:30.8644329+02:00"
+                testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
+                outcome="Failed"
+                testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d"
+                relativeResultsDirectory="268fd488-12c7-4107-80d8-5df1200cd637"
+            />
+            <UnitTestResult
+                executionId="daf369f6-7c54-482d-a12c-68357679bd78"
+                testId="446a0829-8d87-1082-ab45-b2ab9f846325"
+                testName="BestaandeZaakOpenen"
+                computerName="XYZ-BLA-24"
+                duration="00:00:01.7342127"
+                startTime="2024-09-12T11:33:27.3275629+02:00"
+                endTime="2024-09-12T11:33:29.3909874+02:00"
+                testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
+                outcome="Passed"
+                testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d"
+                relativeResultsDirectory="daf369f6-7c54-482d-a12c-68357679bd78"
+            />
+        </Results>
+        <TestDefinitions>
+            <UnitTest name="BestaandeZaakOpenen" id="446a0829-8d87-1082-ab45-b2ab9f846325">
+                <TestCategory>
+                    <TestCategoryItem TestCategory="FeatureTag" />
+                    <TestCategoryItem TestCategory="ScenarioTag1" />
+                    <TestCategoryItem TestCategory="JIRA-224" />
+                </TestCategory>
+                <Execution id="daf369f6-7c54-482d-a12c-68357679bd78" />
+                <TestMethod
+                    codeBase="C:\XYZ\FrontendTests.dll"
+                    adapterTypeName="executor://mstestadapter/v2"
+                    className="ClassName"
+                    name="BestaandeZaakOpenen"
+                />
+            </UnitTest>
+            <UnitTest name="BestaandeZaakOpenen2" id="63eb0c90-d1fc-a21e-6fc0-3974b0cc65db">
+                <TestCategory>
+                    <TestCategoryItem TestCategory="FeatureTag" />
+                    <TestCategoryItem TestCategory="ScenarioTag2" />
+                </TestCategory>
+                <Execution id="268fd488-12c7-4107-80d8-5df1200cd637" />
+                <TestMethod
+                    codeBase="C:\XYZ\FrontendTests.dll"
+                    adapterTypeName="executor://mstestadapter/v2"
+                    className="ClassName"
+                    name="BestaandeZaakOpenen2"
+                />
+            </UnitTest>
+        </TestDefinitions>
+    </TestRun>"""

components/collector/tests/source_collectors/visual_studio_trx/test_source_up_to_dateness.py

@@ -0,0 +1,18 @@
+"""Unit tests for the Visual Studio TRX test report source up-to-dateness collector."""
+
+from collector_utilities.date_time import days_ago, parse_datetime
+
+from .base import VisualStudioTRXCollectorTestCase
+
+
+class VisualStudioTRXSourceUpToDatenessTest(VisualStudioTRXCollectorTestCase):
+    """Unit tests for the source up-to-dateness collector."""
+
+    METRIC_TYPE = "source_up_to_dateness"
+    METRIC_ADDITION = "max"
+
+    async def test_source_up_to_dateness(self):
+        """Test that the source age in days is returned."""
+        response = await self.collect(get_request_text=self.VISUAL_STUDIO_TRX_XML)
+        expected_age = days_ago(parse_datetime("2024-09-12T11:33:30.3272909+02:00"))
+        self.assert_measurement(response, value=str(expected_age))

components/collector/tests/source_collectors/visual_studio_trx/test_tests.py

@@ -0,0 +1,37 @@
+"""Unit tests for the Visual Studio TRX test report tests collector."""
+
+from .base import VisualStudioTRXCollectorTestCase
+
+
+class VisualStudioTRXTestReportTest(VisualStudioTRXCollectorTestCase):
+    """Unit tests for the Visual Studio TRX test report metrics."""
+
+    METRIC_TYPE = "tests"
+
+    def setUp(self):
+        """Extend to set up test data."""
+        super().setUp()
+        self.expected_entities = [
+            {
+                "key": "446a0829-8d87-1082-ab45-b2ab9f846325",
+                "name": "BestaandeZaakOpenen (JIRA-224)",
+                "test_result": "Passed",
+            },
+            {
+                "key": "63eb0c90-d1fc-a21e-6fc0-3974b0cc65db",
+                "name": "BestaandeZaakOpenen2",
+                "test_result": "Failed",
+            },
+        ]
+
+    async def test_tests(self):
+        """Test that the number of tests is returned."""
+        response = await self.collect(get_request_text=self.VISUAL_STUDIO_TRX_XML)
+        self.assert_measurement(response, value="2", total="2", entities=self.expected_entities)
+
+    async def test_failed_tests(self):
+        """Test that the failed tests are returned."""
+        self.set_source_parameter("test_result", ["Failed"])
+        response = await self.collect(get_request_text=self.VISUAL_STUDIO_TRX_XML)
+        entities_for_failed_tests = [entity for entity in self.expected_entities if entity["test_result"] == "Failed"]
+        self.assert_measurement(response, value="1", total="2", entities=entities_for_failed_tests)

components/shared_code/src/shared_data_model/metrics.py

@@ -459,6 +459,7 @@
             "sonarqube",
             "testng",
             "trello",
+            "visual_studio_trx",
         ],
         tags=[Tag.CI],
     ),
@@ -576,7 +577,7 @@
         unit=Unit.TEST_CASES,
         direction=Direction.MORE_IS_BETTER,
         near_target="0",
-        sources=["jenkins_test_report", "jira", "junit", "robot_framework", "testng"],
+        sources=["jenkins_test_report", "jira", "junit", "robot_framework", "testng", "visual_studio_trx"],
         tags=[Tag.TEST_QUALITY],
     ),
     "tests": Metric(
@@ -601,6 +602,7 @@
             "robot_framework_jenkins_plugin",
             "sonarqube",
             "testng",
+            "visual_studio_trx",
         ],
         tags=[Tag.TEST_QUALITY],
     ),

components/shared_code/src/shared_data_model/sources/__init__.py

@@ -39,6 +39,7 @@
 from .testng import TESTNG
 from .trello import TRELLO
 from .trivy import TRIVY_JSON
+from .visual_studio_trx import VISUAL_STUDIO_TRX
 
 SOURCES = {
     "anchore": ANCHORE,
@@ -89,4 +90,5 @@
     "testng": TESTNG,
     "trivy_json": TRIVY_JSON,
     "trello": TRELLO,
+    "visual_studio_trx": VISUAL_STUDIO_TRX,
 }

components/shared_code/src/shared_data_model/sources/quality_time.py

@@ -211,6 +211,7 @@
                 "TestNG",
                 "Trello",
                 "Trivy JSON",
+                "Visual Studio TRX",
             ],
             api_values={
                 "Anchore": "anchore",
@@ -261,6 +262,7 @@
                 "TestNG": "testng",
                 "Trello": "trello",
                 "Trivy JSON": "trivy_json",
+                "Visual Studio TRX": "visual_studio_trx",
             },
             metrics=["metrics"],
         ),

components/shared_code/src/shared_data_model/sources/visual_studio_trx.py

@@ -0,0 +1,66 @@
+"""Visual Studio test result file (.trx) source."""
+
+from pydantic import HttpUrl
+
+from shared_data_model.meta.entity import Color, Entity, EntityAttribute
+from shared_data_model.meta.source import Source
+from shared_data_model.parameters import TestResult, access_parameters
+
+ALL_VISUAL_STUDIO_TRX_METRICS = ["source_up_to_dateness", "test_cases", "tests"]
+
+TEST_ENTITIES = Entity(
+    name="test",
+    attributes=[
+        EntityAttribute(name="Unittest name", key="name"),
+        EntityAttribute(
+            name="Test result",
+            color={
+                "Aborted": Color.NEGATIVE,
+                "Completed": Color.POSITIVE,
+                "Disconnected": Color.NEGATIVE,
+                "Error": Color.NEGATIVE,
+                "Failed": Color.NEGATIVE,
+                "Inconclusive": Color.WARNING,
+                "InProgress": Color.WARNING,
+                "NotExecuted": Color.WARNING,
+                "NotRunnable": Color.WARNING,
+                "Passed": Color.POSITIVE,
+                "PassedButRunAborted": Color.POSITIVE,
+                "Pending": Color.WARNING,
+                "Timeout": Color.NEGATIVE,
+                "Warning": Color.WARNING,
+            },
+        ),
+    ],
+)
+
+VISUAL_STUDIO_TRX = Source(
+    name="Visual Studio TRX",
+    description="Test reports in the Visual Studio TRX format.",
+    url=HttpUrl(
+        "https://learn.microsoft.com/en-us/dotnet/core/testing/unit-testing-platform-extensions-test-reports#"
+        "visual-studio-test-reports"
+    ),
+    parameters={
+        "test_result": TestResult(
+            values=[
+                "Aborted",
+                "Completed",
+                "Disconnected",
+                "Error",
+                "Failed",
+                "Inconclusive",
+                "InProgress",
+                "NotExecuted",
+                "NotRunnable",
+                "Passed",
+                "PassedButRunAborted",
+                "Pending",
+                "Timeout",
+                "Warning",
+            ],
+        ),
+        **access_parameters(ALL_VISUAL_STUDIO_TRX_METRICS, source_type="Visual Studio TRX", source_type_format="XML"),
+    },
+    entities={"tests": TEST_ENTITIES, "test_cases": TEST_ENTITIES},
+)