From b1a680518c5a2d5d65074af4cc6574df94432803 Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:23:17 +0100 Subject: [PATCH] Docker SSI: report data to FPD (#3525) --- utils/_context/_scenarios/docker_ssi.py | 43 ++++++++++++------ utils/docker_ssi/docker_ssi_definitions.py | 44 +++++++++++-------- utils/docker_ssi/docker_ssi_matrix_builder.py | 12 ++++- utils/docker_ssi/docker_ssi_model.py | 4 +- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/utils/_context/_scenarios/docker_ssi.py b/utils/_context/_scenarios/docker_ssi.py index 79dc7b6135..6539972b09 100644 --- a/utils/_context/_scenarios/docker_ssi.py +++ b/utils/_context/_scenarios/docker_ssi.py @@ -16,6 +16,7 @@ _get_client as get_docker_client, ) from utils.docker_ssi.docker_ssi_matrix_utils import resolve_runtime_version +from utils.docker_ssi.docker_ssi_definitions import SupportedImages from utils.tools import logger from utils.virtual_machine.vm_logger import vm_logger @@ -39,6 +40,8 @@ def __init__(self, name, doc, scenario_groups=None) -> None: self._required_containers.append(self._weblog_injection) self.weblog_url = "http://localhost:18080" self._tested_components = {} + # scenario configuration that is going to be reported in the final report + self._configuration = {"app_type": "docker_ssi"} def configure(self, config): assert config.option.ssi_library, "library must be set: java,python,nodejs,dotnet,ruby,php" @@ -60,8 +63,6 @@ def configure(self, config): # The runtime that is installed on the base image (because we installed automatically or because the weblog contains the runtime preinstalled). # the language is the language used by the tested datadog library self._installed_language_runtime = None - # usually base_weblog + base_image + (runtime) + arch - self._weblog_composed_name = None logger.stdout( f"Configuring scenario with: Weblog: [{self._base_weblog}] Library: [{self._library}] Base Image: [{self._base_image}] Arch: [{self._arch}] Runtime: [{self._installable_runtime}]" @@ -96,8 +97,8 @@ def configure(self, config): # Extract version of the components that we are testing. json_tested_component = self.ssi_image_builder.tested_components() self.fill_context(json_tested_component) + self.print_installed_components() - self._weblog_composed_name = f"{self._base_weblog}_{self.ssi_image_builder.get_base_docker_tag()}" for container in self._required_containers: try: container.configure(self.replay) @@ -141,21 +142,37 @@ def close_targets(self): def fill_context(self, json_tested_components): """ After extract the components from the weblog, fill the context with the data """ - logger.stdout("\nInstalled components:\n") + image_internal_name = SupportedImages().get_internal_name_from_base_image(self._base_image, self._arch) + self.configuration["os"] = image_internal_name + self.configuration["arch"] = self._arch.replace("linux/", "") for key in json_tested_components: + self._tested_components[key] = json_tested_components[key].lstrip(" ") if key == "weblog_url" and json_tested_components[key]: self.weblog_url = json_tested_components[key].lstrip(" ") continue if key == "runtime_version" and json_tested_components[key]: self._installed_language_runtime = Version(json_tested_components[key].lstrip(" ")) + # Runtime version is stored as configuration not as dependency + del self._tested_components[key] + self.configuration["runtime_version"] = f"{self._installed_language_runtime}" if key.startswith("datadog-apm-inject") and json_tested_components[key]: self._datadog_apm_inject_version = f"v{json_tested_components[key].lstrip(' ')}" - if key.startswith("datadog-apm-library-") and json_tested_components[key]: + if key.startswith("datadog-apm-library-") and self._tested_components[key]: library_version_number = json_tested_components[key].lstrip(" ") self._libray_version = LibraryVersion(self._library, library_version_number) - self._tested_components[key] = json_tested_components[key].lstrip(" ") - logger.stdout(f"{key}: {self._tested_components[key]}") + # We store without the lang sufix + self._tested_components["datadog-apm-library"] = self._tested_components[key] + del self._tested_components[key] + + def print_installed_components(self): + logger.stdout("Installed components") + for component in self.components: + logger.stdout(f"{component}: {self.components[component]}") + + logger.stdout("Configuration") + for conf in self.configuration: + logger.stdout(f"{conf}: {self.configuration[conf]}") def post_setup(self): logger.stdout("--- Waiting for all traces and telemetry to be sent to test agent ---") @@ -182,12 +199,16 @@ def components(self): @property def weblog_variant(self): - return self._weblog_composed_name + return self._base_weblog @property def dd_apm_inject_version(self): return self._datadog_apm_inject_version + @property + def configuration(self): + return self._configuration + class DockerSSIImageBuilder: """ Manages the docker image building for the SSI scenario """ @@ -345,9 +366,7 @@ def build_weblog_image(self, ssi_installer_docker_tag): buildargs={"DD_LANG": self._library, "BASE_IMAGE": ssi_installer_docker_tag}, ) self.print_docker_build_logs(self.ssi_all_docker_tag, build_logs) - logger.stdout( - f"0000[tag:{weblog_docker_tag}] Building weblog app on base image [{self.ssi_all_docker_tag}]." - ) + logger.stdout(f"[tag:{weblog_docker_tag}] Building weblog app on base image [{self.ssi_all_docker_tag}].") # Build the weblog image self._weblog_docker_image, build_logs = get_docker_client().images.build( path=".", @@ -357,7 +376,6 @@ def build_weblog_image(self, ssi_installer_docker_tag): nocache=self._force_build or self.should_push_base_images, buildargs={"BASE_IMAGE": self.ssi_all_docker_tag}, ) - logger.info("Weblog build done 000000000!") self.print_docker_build_logs(weblog_docker_tag, build_logs) logger.info("Weblog build done!") except BuildError as e: @@ -385,7 +403,6 @@ def print_docker_build_logs(self, image_tag, build_logs): vm_logger(scenario_name, "docker_build").info("***************************************************************") for chunk in build_logs: - logger.debug("chunk") if "stream" in chunk: for line in chunk["stream"].splitlines(): vm_logger(scenario_name, "docker_build").info(line) diff --git a/utils/docker_ssi/docker_ssi_definitions.py b/utils/docker_ssi/docker_ssi_definitions.py index 98586a7a1d..1bf2cd3ddb 100644 --- a/utils/docker_ssi/docker_ssi_definitions.py +++ b/utils/docker_ssi/docker_ssi_definitions.py @@ -11,30 +11,36 @@ class SupportedImages: """ All supported images """ def __init__(self) -> None: - - self.UBUNTU_22_AMD64 = DockerImage("ubuntu:22.04", LINUX_AMD64) - self.UBUNTU_22_ARM64 = DockerImage("ubuntu:22.04", LINUX_ARM64) - self.UBUNTU_16_AMD64 = DockerImage("ubuntu:16.04", LINUX_AMD64) - self.UBUNTU_16_ARM64 = DockerImage("ubuntu:16.04", LINUX_ARM64) - self.CENTOS_7_AMD64 = DockerImage("centos:7", LINUX_AMD64) - self.ORACLELINUX_9_ARM64 = DockerImage("oraclelinux:9", LINUX_ARM64) - self.ORACLELINUX_9_AMD64 = DockerImage("oraclelinux:9", LINUX_AMD64) - self.ORACLELINUX_8_ARM64 = DockerImage("oraclelinux:8.10", LINUX_ARM64) - self.ORACLELINUX_8_AMD64 = DockerImage("oraclelinux:8.10", LINUX_AMD64) - - self.ALMALINUX_9_ARM64 = DockerImage("almalinux:9.4", LINUX_ARM64) - self.ALMALINUX_9_AMD64 = DockerImage("almalinux:9.4", LINUX_AMD64) - self.ALMALINUX_8_ARM64 = DockerImage("almalinux:8.10", LINUX_ARM64) - self.ALMALINUX_8_AMD64 = DockerImage("almalinux:8.10", LINUX_AMD64) + # Try to set the same name as utils/_context/virtual_machines.py + self.UBUNTU_22_AMD64 = DockerImage("Ubuntu_22", "ubuntu:22.04", LINUX_AMD64) + self.UBUNTU_22_ARM64 = DockerImage("Ubuntu_22", "ubuntu:22.04", LINUX_ARM64) + self.UBUNTU_16_AMD64 = DockerImage("Ubuntu_16", "ubuntu:16.04", LINUX_AMD64) + self.UBUNTU_16_ARM64 = DockerImage("Ubuntu_16", "ubuntu:16.04", LINUX_ARM64) + self.CENTOS_7_AMD64 = DockerImage("CentOS_7", "centos:7", LINUX_AMD64) + self.ORACLELINUX_9_ARM64 = DockerImage("OracleLinux_9", "oraclelinux:9", LINUX_ARM64) + self.ORACLELINUX_9_AMD64 = DockerImage("OracleLinux_9", "oraclelinux:9", LINUX_AMD64) + self.ORACLELINUX_8_ARM64 = DockerImage("OracleLinux_8_10", "oraclelinux:8.10", LINUX_ARM64) + self.ORACLELINUX_8_AMD64 = DockerImage("OracleLinux_8_10", "oraclelinux:8.10", LINUX_AMD64) + + self.ALMALINUX_9_ARM64 = DockerImage("AlmaLinux_9", "almalinux:9.4", LINUX_ARM64) + self.ALMALINUX_9_AMD64 = DockerImage("AlmaLinux_9", "almalinux:9.4", LINUX_AMD64) + self.ALMALINUX_8_ARM64 = DockerImage("AlmaLinux_8", "almalinux:8.10", LINUX_ARM64) + self.ALMALINUX_8_AMD64 = DockerImage("AlmaLinux_8", "almalinux:8.10", LINUX_AMD64) # Currently bugged # DockerImage("centos:7", LINUX_ARM64, short_name="centos_7") # DockerImage("alpine:3", LINUX_AMD64, short_name="alpine_3"), # DockerImage("alpine:3", LINUX_ARM64, short_name="alpine_3"), - self.TOMCAT_9_AMD64 = DockerImage("tomcat:9", LINUX_AMD64) - self.TOMCAT_9_ARM64 = DockerImage("tomcat:9", LINUX_ARM64) - self.WEBSPHERE_AMD64 = DockerImage("icr.io/appcafe/websphere-traditional", LINUX_AMD64) - self.JBOSS_AMD64 = DockerImage("quay.io/wildfly/wildfly:26.1.2.Final", LINUX_AMD64) + self.TOMCAT_9_AMD64 = DockerImage("Tomcat_9", "tomcat:9", LINUX_AMD64) + self.TOMCAT_9_ARM64 = DockerImage("Tomcat_9", "tomcat:9", LINUX_ARM64) + self.WEBSPHERE_AMD64 = DockerImage("Websphere", "icr.io/appcafe/websphere-traditional", LINUX_AMD64) + self.JBOSS_AMD64 = DockerImage("Wildfly", "quay.io/wildfly/wildfly:26.1.2.Final", LINUX_AMD64) + + def get_internal_name_from_base_image(self, base_image, arch): + for image in self.__dict__.values(): + if image.tag == base_image and image.platform == arch: + return image.internal_name + raise ValueError(f"Image {base_image} not supported") class JavaRuntimeInstallableVersions: diff --git a/utils/docker_ssi/docker_ssi_matrix_builder.py b/utils/docker_ssi/docker_ssi_matrix_builder.py index a5a7af7708..3947924e30 100644 --- a/utils/docker_ssi/docker_ssi_matrix_builder.py +++ b/utils/docker_ssi/docker_ssi_matrix_builder.py @@ -30,7 +30,17 @@ def generate_gitlab_pipeline(languages): {"if": '$PARENT_PIPELINE_SOURCE == "schedule"', "when": "always"}, {"when": "manual", "allow_failure": True}, ], - "artifacts": {"when": "always", "paths": ["logs_docker_ssi/"]}, + "after_script": [ + 'SCENARIO_SUFIX=$(echo "DOCKER_SSI" | tr "[:upper:]" "[:lower:]")', + 'REPORTS_PATH="reports/"', + 'mkdir -p "$REPORTS_PATH"', + 'cp -R logs_"${SCENARIO_SUFIX}" $REPORTS_PATH/', + 'cleaned_base_image=$(echo "$base_image" | tr -cd "[:alnum:]_")', + 'cleaned_arch=$(echo "$arch" | tr -cd "[:alnum:]_")', + 'cleaned_runtime=$(echo "$installable_runtime" | tr -cd "[:alnum:]_")', + 'mv "$REPORTS_PATH"/logs_"${SCENARIO_SUFIX}" "$REPORTS_PATH"/logs_"${TEST_LIBRARY}"_"${weblog}"_"${SCENARIO_SUFIX}_${cleaned_base_image}_${cleaned_arch}_${cleaned_runtime}"', + ], + "artifacts": {"when": "always", "paths": ["reports/"]}, }, } diff --git a/utils/docker_ssi/docker_ssi_model.py b/utils/docker_ssi/docker_ssi_model.py index f3d3c56de8..979d7bc65b 100644 --- a/utils/docker_ssi/docker_ssi_model.py +++ b/utils/docker_ssi/docker_ssi_model.py @@ -9,7 +9,9 @@ def __init__(self, version_id, version) -> None: class DockerImage: """ Encapsulates information of the docker image """ - def __init__(self, tag, platform) -> None: + def __init__(self, internal_name, tag, platform) -> None: + # Try to set the same name as utils/_context/virtual_machines.py + self.internal_name = internal_name self.tag = tag self.platform = platform self.runtime_versions = []