Docker SSI: report data to FPD (#3525)

utils/_context/_scenarios/docker_ssi.py

@@ -16,6 +16,7 @@
     _get_client as get_docker_client,
 )
 from utils.docker_ssi.docker_ssi_matrix_utils import resolve_runtime_version
+from utils.docker_ssi.docker_ssi_definitions import SupportedImages
 from utils.tools import logger
 from utils.virtual_machine.vm_logger import vm_logger
 
@@ -39,6 +40,8 @@ def __init__(self, name, doc, scenario_groups=None) -> None:
         self._required_containers.append(self._weblog_injection)
         self.weblog_url = "http://localhost:18080"
         self._tested_components = {}
+        # scenario configuration that is going to be reported in the final report
+        self._configuration = {"app_type": "docker_ssi"}
 
     def configure(self, config):
         assert config.option.ssi_library, "library must be set: java,python,nodejs,dotnet,ruby,php"
@@ -60,8 +63,6 @@ def configure(self, config):
         # The runtime that is installed on the base image (because we installed automatically or because the weblog contains the runtime preinstalled).
         # the language is the language used by the tested datadog library
         self._installed_language_runtime = None
-        # usually base_weblog + base_image + (runtime) + arch
-        self._weblog_composed_name = None
 
         logger.stdout(
             f"Configuring scenario with: Weblog: [{self._base_weblog}] Library: [{self._library}] Base Image: [{self._base_image}] Arch: [{self._arch}] Runtime: [{self._installable_runtime}]"
@@ -96,8 +97,8 @@ def configure(self, config):
         # Extract version of the components that we are testing.
         json_tested_component = self.ssi_image_builder.tested_components()
         self.fill_context(json_tested_component)
+        self.print_installed_components()
 
-        self._weblog_composed_name = f"{self._base_weblog}_{self.ssi_image_builder.get_base_docker_tag()}"
         for container in self._required_containers:
             try:
                 container.configure(self.replay)
@@ -141,21 +142,37 @@ def close_targets(self):
     def fill_context(self, json_tested_components):
         """ After extract the components from the weblog, fill the context with the data """
 
-        logger.stdout("\nInstalled components:\n")
+        image_internal_name = SupportedImages().get_internal_name_from_base_image(self._base_image, self._arch)
+        self.configuration["os"] = image_internal_name
+        self.configuration["arch"] = self._arch.replace("linux/", "")
 
         for key in json_tested_components:
+            self._tested_components[key] = json_tested_components[key].lstrip(" ")
             if key == "weblog_url" and json_tested_components[key]:
                 self.weblog_url = json_tested_components[key].lstrip(" ")
                 continue
             if key == "runtime_version" and json_tested_components[key]:
                 self._installed_language_runtime = Version(json_tested_components[key].lstrip(" "))
+                # Runtime version is stored as configuration not as dependency
+                del self._tested_components[key]
+                self.configuration["runtime_version"] = f"{self._installed_language_runtime}"
             if key.startswith("datadog-apm-inject") and json_tested_components[key]:
                 self._datadog_apm_inject_version = f"v{json_tested_components[key].lstrip(' ')}"
-            if key.startswith("datadog-apm-library-") and json_tested_components[key]:
+            if key.startswith("datadog-apm-library-") and self._tested_components[key]:
                 library_version_number = json_tested_components[key].lstrip(" ")
                 self._libray_version = LibraryVersion(self._library, library_version_number)
-            self._tested_components[key] = json_tested_components[key].lstrip(" ")
-            logger.stdout(f"{key}: {self._tested_components[key]}")
+                # We store without the lang sufix
+                self._tested_components["datadog-apm-library"] = self._tested_components[key]
+                del self._tested_components[key]
+
+    def print_installed_components(self):
+        logger.stdout("Installed components")
+        for component in self.components:
+            logger.stdout(f"{component}: {self.components[component]}")
+
+        logger.stdout("Configuration")
+        for conf in self.configuration:
+            logger.stdout(f"{conf}: {self.configuration[conf]}")
 
     def post_setup(self):
         logger.stdout("--- Waiting for all traces and telemetry to be sent to test agent ---")
@@ -182,12 +199,16 @@ def components(self):
 
     @property
     def weblog_variant(self):
-        return self._weblog_composed_name
+        return self._base_weblog
 
     @property
     def dd_apm_inject_version(self):
         return self._datadog_apm_inject_version
 
+    @property
+    def configuration(self):
+        return self._configuration
+
 
 class DockerSSIImageBuilder:
     """ Manages the docker image building for the SSI scenario """
@@ -345,9 +366,7 @@ def build_weblog_image(self, ssi_installer_docker_tag):
                 buildargs={"DD_LANG": self._library, "BASE_IMAGE": ssi_installer_docker_tag},
             )
             self.print_docker_build_logs(self.ssi_all_docker_tag, build_logs)
-            logger.stdout(
-                f"0000[tag:{weblog_docker_tag}] Building weblog app on base image [{self.ssi_all_docker_tag}]."
-            )
+            logger.stdout(f"[tag:{weblog_docker_tag}] Building weblog app on base image [{self.ssi_all_docker_tag}].")
             # Build the weblog image
             self._weblog_docker_image, build_logs = get_docker_client().images.build(
                 path=".",
@@ -357,7 +376,6 @@ def build_weblog_image(self, ssi_installer_docker_tag):
                 nocache=self._force_build or self.should_push_base_images,
                 buildargs={"BASE_IMAGE": self.ssi_all_docker_tag},
             )
-            logger.info("Weblog build done 000000000!")
             self.print_docker_build_logs(weblog_docker_tag, build_logs)
             logger.info("Weblog build done!")
         except BuildError as e:
@@ -385,7 +403,6 @@ def print_docker_build_logs(self, image_tag, build_logs):
         vm_logger(scenario_name, "docker_build").info("***************************************************************")
 
         for chunk in build_logs:
-            logger.debug("chunk")
             if "stream" in chunk:
                 for line in chunk["stream"].splitlines():
                     vm_logger(scenario_name, "docker_build").info(line)

utils/docker_ssi/docker_ssi_definitions.py

@@ -11,30 +11,36 @@ class SupportedImages:
     """ All supported images """
 
     def __init__(self) -> None:
-
-        self.UBUNTU_22_AMD64 = DockerImage("ubuntu:22.04", LINUX_AMD64)
-        self.UBUNTU_22_ARM64 = DockerImage("ubuntu:22.04", LINUX_ARM64)
-        self.UBUNTU_16_AMD64 = DockerImage("ubuntu:16.04", LINUX_AMD64)
-        self.UBUNTU_16_ARM64 = DockerImage("ubuntu:16.04", LINUX_ARM64)
-        self.CENTOS_7_AMD64 = DockerImage("centos:7", LINUX_AMD64)
-        self.ORACLELINUX_9_ARM64 = DockerImage("oraclelinux:9", LINUX_ARM64)
-        self.ORACLELINUX_9_AMD64 = DockerImage("oraclelinux:9", LINUX_AMD64)
-        self.ORACLELINUX_8_ARM64 = DockerImage("oraclelinux:8.10", LINUX_ARM64)
-        self.ORACLELINUX_8_AMD64 = DockerImage("oraclelinux:8.10", LINUX_AMD64)
-
-        self.ALMALINUX_9_ARM64 = DockerImage("almalinux:9.4", LINUX_ARM64)
-        self.ALMALINUX_9_AMD64 = DockerImage("almalinux:9.4", LINUX_AMD64)
-        self.ALMALINUX_8_ARM64 = DockerImage("almalinux:8.10", LINUX_ARM64)
-        self.ALMALINUX_8_AMD64 = DockerImage("almalinux:8.10", LINUX_AMD64)
+        # Try to set the same name as utils/_context/virtual_machines.py
+        self.UBUNTU_22_AMD64 = DockerImage("Ubuntu_22", "ubuntu:22.04", LINUX_AMD64)
+        self.UBUNTU_22_ARM64 = DockerImage("Ubuntu_22", "ubuntu:22.04", LINUX_ARM64)
+        self.UBUNTU_16_AMD64 = DockerImage("Ubuntu_16", "ubuntu:16.04", LINUX_AMD64)
+        self.UBUNTU_16_ARM64 = DockerImage("Ubuntu_16", "ubuntu:16.04", LINUX_ARM64)
+        self.CENTOS_7_AMD64 = DockerImage("CentOS_7", "centos:7", LINUX_AMD64)
+        self.ORACLELINUX_9_ARM64 = DockerImage("OracleLinux_9", "oraclelinux:9", LINUX_ARM64)
+        self.ORACLELINUX_9_AMD64 = DockerImage("OracleLinux_9", "oraclelinux:9", LINUX_AMD64)
+        self.ORACLELINUX_8_ARM64 = DockerImage("OracleLinux_8_10", "oraclelinux:8.10", LINUX_ARM64)
+        self.ORACLELINUX_8_AMD64 = DockerImage("OracleLinux_8_10", "oraclelinux:8.10", LINUX_AMD64)
+
+        self.ALMALINUX_9_ARM64 = DockerImage("AlmaLinux_9", "almalinux:9.4", LINUX_ARM64)
+        self.ALMALINUX_9_AMD64 = DockerImage("AlmaLinux_9", "almalinux:9.4", LINUX_AMD64)
+        self.ALMALINUX_8_ARM64 = DockerImage("AlmaLinux_8", "almalinux:8.10", LINUX_ARM64)
+        self.ALMALINUX_8_AMD64 = DockerImage("AlmaLinux_8", "almalinux:8.10", LINUX_AMD64)
 
         # Currently bugged
         # DockerImage("centos:7", LINUX_ARM64, short_name="centos_7")
         # DockerImage("alpine:3", LINUX_AMD64, short_name="alpine_3"),
         # DockerImage("alpine:3", LINUX_ARM64, short_name="alpine_3"),
-        self.TOMCAT_9_AMD64 = DockerImage("tomcat:9", LINUX_AMD64)
-        self.TOMCAT_9_ARM64 = DockerImage("tomcat:9", LINUX_ARM64)
-        self.WEBSPHERE_AMD64 = DockerImage("icr.io/appcafe/websphere-traditional", LINUX_AMD64)
-        self.JBOSS_AMD64 = DockerImage("quay.io/wildfly/wildfly:26.1.2.Final", LINUX_AMD64)
+        self.TOMCAT_9_AMD64 = DockerImage("Tomcat_9", "tomcat:9", LINUX_AMD64)
+        self.TOMCAT_9_ARM64 = DockerImage("Tomcat_9", "tomcat:9", LINUX_ARM64)
+        self.WEBSPHERE_AMD64 = DockerImage("Websphere", "icr.io/appcafe/websphere-traditional", LINUX_AMD64)
+        self.JBOSS_AMD64 = DockerImage("Wildfly", "quay.io/wildfly/wildfly:26.1.2.Final", LINUX_AMD64)
+
+    def get_internal_name_from_base_image(self, base_image, arch):
+        for image in self.__dict__.values():
+            if image.tag == base_image and image.platform == arch:
+                return image.internal_name
+        raise ValueError(f"Image {base_image} not supported")
 
 
 class JavaRuntimeInstallableVersions:

utils/docker_ssi/docker_ssi_matrix_builder.py

@@ -30,7 +30,17 @@ def generate_gitlab_pipeline(languages):
                 {"if": '$PARENT_PIPELINE_SOURCE == "schedule"', "when": "always"},
                 {"when": "manual", "allow_failure": True},
             ],
-            "artifacts": {"when": "always", "paths": ["logs_docker_ssi/"]},
+            "after_script": [
+                'SCENARIO_SUFIX=$(echo "DOCKER_SSI" | tr "[:upper:]" "[:lower:]")',
+                'REPORTS_PATH="reports/"',
+                'mkdir -p "$REPORTS_PATH"',
+                'cp -R logs_"${SCENARIO_SUFIX}" $REPORTS_PATH/',
+                'cleaned_base_image=$(echo "$base_image" | tr -cd "[:alnum:]_")',
+                'cleaned_arch=$(echo "$arch" | tr -cd "[:alnum:]_")',
+                'cleaned_runtime=$(echo "$installable_runtime" | tr -cd "[:alnum:]_")',
+                'mv "$REPORTS_PATH"/logs_"${SCENARIO_SUFIX}" "$REPORTS_PATH"/logs_"${TEST_LIBRARY}"_"${weblog}"_"${SCENARIO_SUFIX}_${cleaned_base_image}_${cleaned_arch}_${cleaned_runtime}"',
+            ],
+            "artifacts": {"when": "always", "paths": ["reports/"]},
         },
     }
 

utils/docker_ssi/docker_ssi_model.py

@@ -9,7 +9,9 @@ def __init__(self, version_id, version) -> None:
 class DockerImage:
     """ Encapsulates information of the docker image """
 
-    def __init__(self, tag, platform) -> None:
+    def __init__(self, internal_name, tag, platform) -> None:
+        # Try to set the same name as utils/_context/virtual_machines.py
+        self.internal_name = internal_name
         self.tag = tag
         self.platform = platform
         self.runtime_versions = []