From 06e1449cbcbd07e5cf2d0321240f8961f42134b2 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 21 Nov 2024 18:01:29 +0100 Subject: [PATCH 1/6] Docker SSI: report data to FPD --- utils/_context/_scenarios/docker_ssi.py | 32 ++++++++++++++-- utils/docker_ssi/docker_ssi_definitions.py | 44 ++++++++++++---------- utils/docker_ssi/docker_ssi_model.py | 4 +- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/utils/_context/_scenarios/docker_ssi.py b/utils/_context/_scenarios/docker_ssi.py index 79dc7b6135..8a3a82cdfc 100644 --- a/utils/_context/_scenarios/docker_ssi.py +++ b/utils/_context/_scenarios/docker_ssi.py @@ -16,6 +16,7 @@ _get_client as get_docker_client, ) from utils.docker_ssi.docker_ssi_matrix_utils import resolve_runtime_version +from utils.docker_ssi.docker_ssi_definitions import SupportedImages from utils.tools import logger from utils.virtual_machine.vm_logger import vm_logger @@ -39,6 +40,8 @@ def __init__(self, name, doc, scenario_groups=None) -> None: self._required_containers.append(self._weblog_injection) self.weblog_url = "http://localhost:18080" self._tested_components = {} + # scenario configuration that is going to be reported in the final report + self._configuration = {"app_type": "docker_ssi"} def configure(self, config): assert config.option.ssi_library, "library must be set: java,python,nodejs,dotnet,ruby,php" @@ -96,6 +99,7 @@ def configure(self, config): # Extract version of the components that we are testing. json_tested_component = self.ssi_image_builder.tested_components() self.fill_context(json_tested_component) + self.print_installed_components() self._weblog_composed_name = f"{self._base_weblog}_{self.ssi_image_builder.get_base_docker_tag()}" for container in self._required_containers: @@ -141,21 +145,37 @@ def close_targets(self): def fill_context(self, json_tested_components): """ After extract the components from the weblog, fill the context with the data """ - logger.stdout("\nInstalled components:\n") + image_internal_name = SupportedImages().get_internal_name_from_base_image(self._base_image, self._arch) + self.configuration["os"] = image_internal_name + self.configuration["arch"] = self._arch for key in json_tested_components: + self._tested_components[key] = json_tested_components[key].lstrip(" ") if key == "weblog_url" and json_tested_components[key]: self.weblog_url = json_tested_components[key].lstrip(" ") continue if key == "runtime_version" and json_tested_components[key]: self._installed_language_runtime = Version(json_tested_components[key].lstrip(" ")) + # Runtime version is stored as configuration not as dependency + del self._tested_components[key] + self.configuration["runtime_version"] = self._installed_language_runtime if key.startswith("datadog-apm-inject") and json_tested_components[key]: self._datadog_apm_inject_version = f"v{json_tested_components[key].lstrip(' ')}" - if key.startswith("datadog-apm-library-") and json_tested_components[key]: + if key.startswith("datadog-apm-library-") and self._tested_components[key]: library_version_number = json_tested_components[key].lstrip(" ") self._libray_version = LibraryVersion(self._library, library_version_number) - self._tested_components[key] = json_tested_components[key].lstrip(" ") - logger.stdout(f"{key}: {self._tested_components[key]}") + # We store without the lang sufix + self._tested_components["datadog-apm-library"] = self._tested_components[key] + del self._tested_components[key] + + def print_installed_components(self): + logger.terminal.write_sep("=", "Installed components", bold=True) + for component in self.components: + logger.stdout(f"{component}: {self.components[component]}") + + logger.terminal.write_sep("=", "Configuration", bold=True) + for conf in self.configuration: + logger.stdout(f"{conf}: {self.configuration[conf]}") def post_setup(self): logger.stdout("--- Waiting for all traces and telemetry to be sent to test agent ---") @@ -188,6 +208,10 @@ def weblog_variant(self): def dd_apm_inject_version(self): return self._datadog_apm_inject_version + @property + def configuration(self): + return self._configuration + class DockerSSIImageBuilder: """ Manages the docker image building for the SSI scenario """ diff --git a/utils/docker_ssi/docker_ssi_definitions.py b/utils/docker_ssi/docker_ssi_definitions.py index 98586a7a1d..1bf2cd3ddb 100644 --- a/utils/docker_ssi/docker_ssi_definitions.py +++ b/utils/docker_ssi/docker_ssi_definitions.py @@ -11,30 +11,36 @@ class SupportedImages: """ All supported images """ def __init__(self) -> None: - - self.UBUNTU_22_AMD64 = DockerImage("ubuntu:22.04", LINUX_AMD64) - self.UBUNTU_22_ARM64 = DockerImage("ubuntu:22.04", LINUX_ARM64) - self.UBUNTU_16_AMD64 = DockerImage("ubuntu:16.04", LINUX_AMD64) - self.UBUNTU_16_ARM64 = DockerImage("ubuntu:16.04", LINUX_ARM64) - self.CENTOS_7_AMD64 = DockerImage("centos:7", LINUX_AMD64) - self.ORACLELINUX_9_ARM64 = DockerImage("oraclelinux:9", LINUX_ARM64) - self.ORACLELINUX_9_AMD64 = DockerImage("oraclelinux:9", LINUX_AMD64) - self.ORACLELINUX_8_ARM64 = DockerImage("oraclelinux:8.10", LINUX_ARM64) - self.ORACLELINUX_8_AMD64 = DockerImage("oraclelinux:8.10", LINUX_AMD64) - - self.ALMALINUX_9_ARM64 = DockerImage("almalinux:9.4", LINUX_ARM64) - self.ALMALINUX_9_AMD64 = DockerImage("almalinux:9.4", LINUX_AMD64) - self.ALMALINUX_8_ARM64 = DockerImage("almalinux:8.10", LINUX_ARM64) - self.ALMALINUX_8_AMD64 = DockerImage("almalinux:8.10", LINUX_AMD64) + # Try to set the same name as utils/_context/virtual_machines.py + self.UBUNTU_22_AMD64 = DockerImage("Ubuntu_22", "ubuntu:22.04", LINUX_AMD64) + self.UBUNTU_22_ARM64 = DockerImage("Ubuntu_22", "ubuntu:22.04", LINUX_ARM64) + self.UBUNTU_16_AMD64 = DockerImage("Ubuntu_16", "ubuntu:16.04", LINUX_AMD64) + self.UBUNTU_16_ARM64 = DockerImage("Ubuntu_16", "ubuntu:16.04", LINUX_ARM64) + self.CENTOS_7_AMD64 = DockerImage("CentOS_7", "centos:7", LINUX_AMD64) + self.ORACLELINUX_9_ARM64 = DockerImage("OracleLinux_9", "oraclelinux:9", LINUX_ARM64) + self.ORACLELINUX_9_AMD64 = DockerImage("OracleLinux_9", "oraclelinux:9", LINUX_AMD64) + self.ORACLELINUX_8_ARM64 = DockerImage("OracleLinux_8_10", "oraclelinux:8.10", LINUX_ARM64) + self.ORACLELINUX_8_AMD64 = DockerImage("OracleLinux_8_10", "oraclelinux:8.10", LINUX_AMD64) + + self.ALMALINUX_9_ARM64 = DockerImage("AlmaLinux_9", "almalinux:9.4", LINUX_ARM64) + self.ALMALINUX_9_AMD64 = DockerImage("AlmaLinux_9", "almalinux:9.4", LINUX_AMD64) + self.ALMALINUX_8_ARM64 = DockerImage("AlmaLinux_8", "almalinux:8.10", LINUX_ARM64) + self.ALMALINUX_8_AMD64 = DockerImage("AlmaLinux_8", "almalinux:8.10", LINUX_AMD64) # Currently bugged # DockerImage("centos:7", LINUX_ARM64, short_name="centos_7") # DockerImage("alpine:3", LINUX_AMD64, short_name="alpine_3"), # DockerImage("alpine:3", LINUX_ARM64, short_name="alpine_3"), - self.TOMCAT_9_AMD64 = DockerImage("tomcat:9", LINUX_AMD64) - self.TOMCAT_9_ARM64 = DockerImage("tomcat:9", LINUX_ARM64) - self.WEBSPHERE_AMD64 = DockerImage("icr.io/appcafe/websphere-traditional", LINUX_AMD64) - self.JBOSS_AMD64 = DockerImage("quay.io/wildfly/wildfly:26.1.2.Final", LINUX_AMD64) + self.TOMCAT_9_AMD64 = DockerImage("Tomcat_9", "tomcat:9", LINUX_AMD64) + self.TOMCAT_9_ARM64 = DockerImage("Tomcat_9", "tomcat:9", LINUX_ARM64) + self.WEBSPHERE_AMD64 = DockerImage("Websphere", "icr.io/appcafe/websphere-traditional", LINUX_AMD64) + self.JBOSS_AMD64 = DockerImage("Wildfly", "quay.io/wildfly/wildfly:26.1.2.Final", LINUX_AMD64) + + def get_internal_name_from_base_image(self, base_image, arch): + for image in self.__dict__.values(): + if image.tag == base_image and image.platform == arch: + return image.internal_name + raise ValueError(f"Image {base_image} not supported") class JavaRuntimeInstallableVersions: diff --git a/utils/docker_ssi/docker_ssi_model.py b/utils/docker_ssi/docker_ssi_model.py index f3d3c56de8..979d7bc65b 100644 --- a/utils/docker_ssi/docker_ssi_model.py +++ b/utils/docker_ssi/docker_ssi_model.py @@ -9,7 +9,9 @@ def __init__(self, version_id, version) -> None: class DockerImage: """ Encapsulates information of the docker image """ - def __init__(self, tag, platform) -> None: + def __init__(self, internal_name, tag, platform) -> None: + # Try to set the same name as utils/_context/virtual_machines.py + self.internal_name = internal_name self.tag = tag self.platform = platform self.runtime_versions = [] From 1945ba430aca43c0a72479c5281001a644d0362c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Fri, 22 Nov 2024 08:11:06 +0100 Subject: [PATCH 2/6] remove terminal out --- utils/_context/_scenarios/docker_ssi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/_context/_scenarios/docker_ssi.py b/utils/_context/_scenarios/docker_ssi.py index 8a3a82cdfc..28f397ffca 100644 --- a/utils/_context/_scenarios/docker_ssi.py +++ b/utils/_context/_scenarios/docker_ssi.py @@ -169,11 +169,11 @@ def fill_context(self, json_tested_components): del self._tested_components[key] def print_installed_components(self): - logger.terminal.write_sep("=", "Installed components", bold=True) + logger.stdout("Installed components") for component in self.components: logger.stdout(f"{component}: {self.components[component]}") - logger.terminal.write_sep("=", "Configuration", bold=True) + logger.stdout("Configuration") for conf in self.configuration: logger.stdout(f"{conf}: {self.configuration[conf]}") From c9dd36a887c0ac26a730058363ff8939b8b936b9 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Fri, 22 Nov 2024 09:06:24 +0100 Subject: [PATCH 3/6] fix runtime version --- utils/_context/_scenarios/docker_ssi.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/utils/_context/_scenarios/docker_ssi.py b/utils/_context/_scenarios/docker_ssi.py index 28f397ffca..e89bda8380 100644 --- a/utils/_context/_scenarios/docker_ssi.py +++ b/utils/_context/_scenarios/docker_ssi.py @@ -158,7 +158,7 @@ def fill_context(self, json_tested_components): self._installed_language_runtime = Version(json_tested_components[key].lstrip(" ")) # Runtime version is stored as configuration not as dependency del self._tested_components[key] - self.configuration["runtime_version"] = self._installed_language_runtime + self.configuration["runtime_version"] = f"{self._installed_language_runtime}" if key.startswith("datadog-apm-inject") and json_tested_components[key]: self._datadog_apm_inject_version = f"v{json_tested_components[key].lstrip(' ')}" if key.startswith("datadog-apm-library-") and self._tested_components[key]: @@ -369,9 +369,7 @@ def build_weblog_image(self, ssi_installer_docker_tag): buildargs={"DD_LANG": self._library, "BASE_IMAGE": ssi_installer_docker_tag}, ) self.print_docker_build_logs(self.ssi_all_docker_tag, build_logs) - logger.stdout( - f"0000[tag:{weblog_docker_tag}] Building weblog app on base image [{self.ssi_all_docker_tag}]." - ) + logger.stdout(f"[tag:{weblog_docker_tag}] Building weblog app on base image [{self.ssi_all_docker_tag}].") # Build the weblog image self._weblog_docker_image, build_logs = get_docker_client().images.build( path=".", @@ -381,7 +379,6 @@ def build_weblog_image(self, ssi_installer_docker_tag): nocache=self._force_build or self.should_push_base_images, buildargs={"BASE_IMAGE": self.ssi_all_docker_tag}, ) - logger.info("Weblog build done 000000000!") self.print_docker_build_logs(weblog_docker_tag, build_logs) logger.info("Weblog build done!") except BuildError as e: @@ -409,7 +406,6 @@ def print_docker_build_logs(self, image_tag, build_logs): vm_logger(scenario_name, "docker_build").info("***************************************************************") for chunk in build_logs: - logger.debug("chunk") if "stream" in chunk: for line in chunk["stream"].splitlines(): vm_logger(scenario_name, "docker_build").info(line) From 17753a68e0631bdac867612f57403e7600b76a9e Mon Sep 17 00:00:00 2001 From: roberto montero Date: Fri, 22 Nov 2024 10:28:21 +0100 Subject: [PATCH 4/6] archive logs gitlab --- utils/_context/_scenarios/docker_ssi.py | 2 +- utils/docker_ssi/docker_ssi_matrix_builder.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/utils/_context/_scenarios/docker_ssi.py b/utils/_context/_scenarios/docker_ssi.py index e89bda8380..3008d0dfe6 100644 --- a/utils/_context/_scenarios/docker_ssi.py +++ b/utils/_context/_scenarios/docker_ssi.py @@ -147,7 +147,7 @@ def fill_context(self, json_tested_components): image_internal_name = SupportedImages().get_internal_name_from_base_image(self._base_image, self._arch) self.configuration["os"] = image_internal_name - self.configuration["arch"] = self._arch + self.configuration["arch"] = self._arch.replace("linux/", "") for key in json_tested_components: self._tested_components[key] = json_tested_components[key].lstrip(" ") diff --git a/utils/docker_ssi/docker_ssi_matrix_builder.py b/utils/docker_ssi/docker_ssi_matrix_builder.py index a5a7af7708..84a3843b3d 100644 --- a/utils/docker_ssi/docker_ssi_matrix_builder.py +++ b/utils/docker_ssi/docker_ssi_matrix_builder.py @@ -30,7 +30,16 @@ def generate_gitlab_pipeline(languages): {"if": '$PARENT_PIPELINE_SOURCE == "schedule"', "when": "always"}, {"when": "manual", "allow_failure": True}, ], - "artifacts": {"when": "always", "paths": ["logs_docker_ssi/"]}, + "after_script": [ + 'SCENARIO_SUFIX=$(echo "DOCKER_SSI" | tr "[:upper:]" "[:lower:]")', + 'REPORTS_PATH="reports/"', + 'mkdir -p "$REPORTS_PATH"', + 'cp -R logs_"${SCENARIO_SUFIX}" $REPORTS_PATH/', + 'cleaned_base_image=$(echo "$base_image" | tr -cd "[:alnum:]_")', + 'cleaned_arch=$(echo "$arch" | tr -cd "[:alnum:]_")', + 'mv "$REPORTS_PATH"/logs_"${SCENARIO_SUFIX}" "$REPORTS_PATH"/logs_"${TEST_LIBRARY}"_"${weblog}"_"${SCENARIO_SUFIX}_${cleaned_base_image}_${cleaned_arch}"', + ], + "artifacts": {"when": "always", "paths": ["reports/"]}, }, } From b3b0808e71562f98ea7eb92fa75b07bd26df5739 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Fri, 22 Nov 2024 10:47:44 +0100 Subject: [PATCH 5/6] add instalablerutime to report folder name --- utils/docker_ssi/docker_ssi_matrix_builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/docker_ssi/docker_ssi_matrix_builder.py b/utils/docker_ssi/docker_ssi_matrix_builder.py index 84a3843b3d..3947924e30 100644 --- a/utils/docker_ssi/docker_ssi_matrix_builder.py +++ b/utils/docker_ssi/docker_ssi_matrix_builder.py @@ -37,7 +37,8 @@ def generate_gitlab_pipeline(languages): 'cp -R logs_"${SCENARIO_SUFIX}" $REPORTS_PATH/', 'cleaned_base_image=$(echo "$base_image" | tr -cd "[:alnum:]_")', 'cleaned_arch=$(echo "$arch" | tr -cd "[:alnum:]_")', - 'mv "$REPORTS_PATH"/logs_"${SCENARIO_SUFIX}" "$REPORTS_PATH"/logs_"${TEST_LIBRARY}"_"${weblog}"_"${SCENARIO_SUFIX}_${cleaned_base_image}_${cleaned_arch}"', + 'cleaned_runtime=$(echo "$installable_runtime" | tr -cd "[:alnum:]_")', + 'mv "$REPORTS_PATH"/logs_"${SCENARIO_SUFIX}" "$REPORTS_PATH"/logs_"${TEST_LIBRARY}"_"${weblog}"_"${SCENARIO_SUFIX}_${cleaned_base_image}_${cleaned_arch}_${cleaned_runtime}"', ], "artifacts": {"when": "always", "paths": ["reports/"]}, }, From 1e21e66331644cd744ad8d9dc7348481e5eec561 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Fri, 22 Nov 2024 11:07:27 +0100 Subject: [PATCH 6/6] set weblog name --- utils/_context/_scenarios/docker_ssi.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/_context/_scenarios/docker_ssi.py b/utils/_context/_scenarios/docker_ssi.py index 3008d0dfe6..6539972b09 100644 --- a/utils/_context/_scenarios/docker_ssi.py +++ b/utils/_context/_scenarios/docker_ssi.py @@ -63,8 +63,6 @@ def configure(self, config): # The runtime that is installed on the base image (because we installed automatically or because the weblog contains the runtime preinstalled). # the language is the language used by the tested datadog library self._installed_language_runtime = None - # usually base_weblog + base_image + (runtime) + arch - self._weblog_composed_name = None logger.stdout( f"Configuring scenario with: Weblog: [{self._base_weblog}] Library: [{self._library}] Base Image: [{self._base_image}] Arch: [{self._arch}] Runtime: [{self._installable_runtime}]" @@ -101,7 +99,6 @@ def configure(self, config): self.fill_context(json_tested_component) self.print_installed_components() - self._weblog_composed_name = f"{self._base_weblog}_{self.ssi_image_builder.get_base_docker_tag()}" for container in self._required_containers: try: container.configure(self.replay) @@ -202,7 +199,7 @@ def components(self): @property def weblog_variant(self): - return self._weblog_composed_name + return self._base_weblog @property def dd_apm_inject_version(self):