diff --git a/DEV_GUIDE.md b/DEV_GUIDE.md index b0ac80de2e..b9b13bbd16 100644 --- a/DEV_GUIDE.md +++ b/DEV_GUIDE.md @@ -336,7 +336,7 @@ has been applied ineffectively. * `KROXYLICIOUS_IMAGE_REPO`: url to the image of kroxylicious to be used. Default value: `quay.io/kroxylicious/kroxylicious` * `KROXYLICIOUS_VERSION`: version of kroxylicious to be used. Default value: `${project.version}` in pom file * `KAFKA_VERSION`: kafka version to be used. Default value: `${kafka.version}` in pom file -* `STRIMZI_URL`: url where to download strimzi. Default value: `https://strimzi.io/install/latest?namespace=kafka` +* `STRIMZI_VERSION`: strimzi version to be used. Default value: `${strimzi.version}` in pom file * `SKIP_TEARDOWN`: variable for development purposes to avoid keep deploying and deleting deployments each run. Default value: `false` * `CONTAINER_CONFIG_PATH`: directory where `config.json` file is located. This file contains the pull secrets to be used by the container engine. Default value: `$HOME/.docker/config.json` diff --git a/kroxylicious-systemtests/pom.xml b/kroxylicious-systemtests/pom.xml index ff69af0c58..d929d4d361 100644 --- a/kroxylicious-systemtests/pom.xml +++ b/kroxylicious-systemtests/pom.xml @@ -32,10 +32,6 @@ com.fasterxml.jackson.core jackson-databind - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - io.fabric8 kubernetes-client-api diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java index 243db6ecf5..3dface614a 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java @@ -26,13 +26,13 @@ private Environment() { private static final String KAFKA_VERSION_ENV = "KAFKA_VERSION"; private static final String KROXY_VERSION_ENV = "KROXYLICIOUS_VERSION"; private static final String KROXY_IMAGE_REPO_ENV = "KROXYLICIOUS_IMAGE_REPO"; - private static final String STRIMZI_URL_ENV = "STRIMZI_URL"; private static final String SKIP_TEARDOWN_ENV = "SKIP_TEARDOWN"; public static final String STRIMZI_FEATURE_GATES_ENV = "STRIMZI_FEATURE_GATES"; private static final String CONTAINER_CONFIG_PATH_ENV = "CONTAINER_CONFIG_PATH"; private static final String VAULT_CHART_VERSION_ENV = "VAULT_CHART_VERSION"; private static final String SKIP_STRIMZI_INSTALL_ENV = "SKIP_STRIMZI_INSTALL"; private static final String KAFKA_CLIENT_ENV = "KAFKA_CLIENT"; + private static final String STRIMZI_VERSION_ENV = "STRIMZI_VERSION"; /** * The kafka version default value @@ -53,14 +53,19 @@ private Environment() { } /** - * The url where kroxylicious image lives to be downloaded. + * The kafka version default value */ - private static final String KROXY_IMAGE_REPO_DEFAULT = "quay.io/kroxylicious/kroxylicious"; + private static final String STRIMZI_VERSION_DEFAULT; + + static { + STRIMZI_VERSION_DEFAULT = determineStrimziVersion(); + } /** - * The strimzi installation url for kubernetes. + * The url where kroxylicious image lives to be downloaded. */ - private static final String STRIMZI_URL_DEFAULT = "https://strimzi.io/install/latest?namespace=" + Constants.KAFKA_DEFAULT_NAMESPACE; + private static final String KROXY_IMAGE_REPO_DEFAULT = "quay.io/kroxylicious/kroxylicious"; + /** * The default value for skipping the teardown locally. */ @@ -80,10 +85,7 @@ private Environment() { * KROXY_VERSION env variable assignment */ public static final String KROXY_VERSION = getOrDefault(KROXY_VERSION_ENV, KROXY_VERSION_DEFAULT); - /** - * STRIMZI_URL env variable assignment - */ - public static final String STRIMZI_URL = getOrDefault(STRIMZI_URL_ENV, STRIMZI_URL_DEFAULT); + /** * KROXY_IMAGE_REPO env variable assignment */ @@ -103,6 +105,8 @@ private Environment() { public static final String KAFKA_CLIENT = getOrDefault(KAFKA_CLIENT_ENV, KAFKA_CLIENT_DEFAULT); + public static final String STRIMZI_VERSION = getOrDefault(STRIMZI_VERSION_ENV, STRIMZI_VERSION_DEFAULT); + private static String getOrDefault(String varName, String defaultValue) { return getOrDefault(varName, String::toString, defaultValue); } @@ -139,4 +143,8 @@ private static String determineKroxyliciousVersion() { private static String determineKafkaVersion() { return readMetadataProperty("kafka.version"); } + + private static String determineStrimziVersion() { + return readMetadataProperty("strimzi.version"); + } } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java index 511cd5a4f4..db0d9255c8 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java @@ -6,30 +6,16 @@ package io.kroxylicious.systemtests.installation.strimzi; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InvalidObjectException; -import java.util.List; +import java.nio.file.Path; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; - import io.kroxylicious.systemtests.Constants; import io.kroxylicious.systemtests.Environment; -import io.kroxylicious.systemtests.utils.DeploymentUtils; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; import io.kroxylicious.systemtests.utils.TestUtils; import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; @@ -39,60 +25,18 @@ */ public class Strimzi { private static final Logger LOGGER = LoggerFactory.getLogger(Strimzi.class); + + public static final String STRIMZI_HELM_REPOSITORY = "quay.io/strimzi-helm/strimzi-kafka-operator"; + public static final String STRIMZI_SERVICE_NAME = "strimzi-cluster-operator"; private final String deploymentNamespace; - private final NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable deployment; /** * Instantiates a new Strimzi. * * @param deploymentNamespace the deployment namespace */ - public Strimzi(String deploymentNamespace) throws IOException { + public Strimzi(String deploymentNamespace) { this.deploymentNamespace = deploymentNamespace; - InputStream strimziMultiDocumentYaml = DeploymentUtils.getDeploymentFileFromURL(Environment.STRIMZI_URL); - InputStream strimziMultiDocumentYamlKRaft = configureKRaftModeForStrimzi(strimziMultiDocumentYaml); - deployment = kubeClient().getClient().load(strimziMultiDocumentYamlKRaft); - } - - private InputStream configureKRaftModeForStrimzi(InputStream strimziMultiDocumentYaml) throws IOException { - YAMLFactory yamlFactory = new YAMLFactory(); - ObjectMapper mapper = new YAMLMapper(); - - YAMLParser multiDocumentYamlParser = yamlFactory.createParser(strimziMultiDocumentYaml); - List docs = mapper - .readValues(multiDocumentYamlParser, new TypeReference() { - }) - .readAll(); - boolean found = false; - List deploymentDocs = docs.stream().filter(p -> p.at("/kind").asText("").equals(Constants.DEPLOYMENT)).toList(); - for (JsonNode doc : deploymentDocs) { - ArrayNode arrayNode = (ArrayNode) doc.at("/spec/template/spec/containers"); - ArrayNode envNode = (ArrayNode) arrayNode.get(0).at("/env"); - for (JsonNode node : envNode) { - if (node.at("/name").asText().equals(Environment.STRIMZI_FEATURE_GATES_ENV)) { - found = true; - String value = node.at("/value").asText(); - if (value.isEmpty() || value.isBlank()) { - value = String.join(",", Constants.USE_KRAFT_MODE, Constants.USE_KAFKA_NODE_POOLS); - } - else { - value = value.replace(Constants.DONT_USE_KRAFT_MODE, Constants.USE_KRAFT_MODE) - .replace(Constants.DONT_USE_KAFKA_NODE_POOLS, Constants.USE_KAFKA_NODE_POOLS); - - value = TestUtils.concatStringIfValueDontExist(value, Constants.USE_KRAFT_MODE, ","); - value = TestUtils.concatStringIfValueDontExist(value, Constants.USE_KAFKA_NODE_POOLS, ","); - } - ((ObjectNode) node).put("value", value); - break; - } - } - } - - if (!found) { - throw new InvalidObjectException("STRIMZI_FEATURE_GATES env variable not found in yaml!"); - } - - return new ByteArrayInputStream(mapper.writeValueAsBytes(docs)); } /** @@ -105,8 +49,11 @@ public void deploy() { LOGGER.warn("Skipping strimzi deployment. It is already deployed!"); return; } - deployment.inNamespace(deploymentNamespace).create(); - DeploymentUtils.waitForDeploymentReady(deploymentNamespace, Constants.STRIMZI_DEPLOYMENT_NAME); + + ResourceManager.helmClient().namespace(deploymentNamespace).installByContainerImage(STRIMZI_HELM_REPOSITORY, STRIMZI_SERVICE_NAME, + Optional.of(Environment.STRIMZI_VERSION), + Optional.of(Path.of(TestUtils.getResourcesURI("helm_strimzi_overrides.yaml"))), + Optional.empty()); } /** @@ -119,7 +66,6 @@ public void delete() throws IOException { return; } LOGGER.info("Deleting Strimzi in {} namespace", deploymentNamespace); - deployment.inNamespace(deploymentNamespace).delete(); - DeploymentUtils.waitForDeploymentDeletion(deploymentNamespace, Constants.STRIMZI_DEPLOYMENT_NAME); + ResourceManager.helmClient().namespace(deploymentNamespace).delete(STRIMZI_SERVICE_NAME); } } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/vault/Vault.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/vault/Vault.java index 47a370b3b2..d33da7b3cc 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/vault/Vault.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/vault/Vault.java @@ -9,7 +9,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.URISyntaxException; import java.nio.file.Path; import java.time.Duration; import java.util.Map; @@ -25,8 +24,7 @@ import io.kroxylicious.systemtests.resources.manager.ResourceManager; import io.kroxylicious.systemtests.utils.DeploymentUtils; import io.kroxylicious.systemtests.utils.NamespaceUtils; - -import edu.umd.cs.findbugs.annotations.NonNull; +import io.kroxylicious.systemtests.utils.TestUtils; import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; @@ -136,30 +134,13 @@ public void deploy() { ResourceManager.helmClient().addRepository(VAULT_HELM_REPOSITORY_NAME, VAULT_HELM_REPOSITORY_URL); ResourceManager.helmClient().namespace(deploymentNamespace).install(VAULT_HELM_CHART_NAME, VAULT_SERVICE_NAME, Optional.of(Environment.VAULT_CHART_VERSION), - Optional.of(getHelmOverridePath()), + Optional.of(Path.of(TestUtils.getResourcesURI("helm_vault_overrides.yaml"))), Optional.of(Map.of("server.dev.devRootToken", vaultRootToken, "global.openshift", String.valueOf(openshiftCluster)))); DeploymentUtils.waitForDeploymentRunning(deploymentNamespace, VAULT_POD_NAME, Duration.ofMinutes(1)); } - @NonNull - private Path getHelmOverridePath() { - var name = "helm_vault_overrides.yaml"; - Path overrideFile; - var resource = getClass().getResource(name); - try { - if (resource == null) { - throw new IllegalStateException("Cannot find override resource " + name + " on classpath"); - } - overrideFile = Path.of(resource.toURI()); - } - catch (URISyntaxException e) { - throw new IllegalStateException("Cannot determine file system path for " + resource); - } - return overrideFile; - } - /** * Delete. * diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/HelmClient.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/HelmClient.java index 8e9220072a..58998f33d2 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/HelmClient.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/HelmClient.java @@ -98,6 +98,11 @@ public HelmClient install(String chartName, String releaseName, Optional return this; } + public HelmClient installByContainerImage(String helmRepositoryOci, String releaseName, Optional version, Optional overrideFile, + Optional> overrideMap) { + return install("oci://" + helmRepositoryOci, releaseName, version, overrideFile, overrideMap); + } + /** * Add repository to the helm client. * @@ -119,7 +124,7 @@ public HelmClient addRepository(String repoName, String repoUrl) { */ public HelmClient delete(String releaseName) { LOGGER.info("Deleting helm-chart {}", releaseName); - delete(String.valueOf(namespace), releaseName); + delete(namespace.orElse("default"), releaseName); return this; } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java index 605508184d..a3890e16f7 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java @@ -6,16 +6,23 @@ package io.kroxylicious.systemtests.utils; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; +import edu.umd.cs.findbugs.annotations.NonNull; + /** * The type Test utils. */ public class TestUtils { + private TestUtils() { + } + /** * Gets default posix file permissions. * @@ -26,21 +33,24 @@ public static FileAttribute> getDefaultPosixFilePermiss } /** - * Concat string if value don't exist. + * Gets resources URI. * - * @param originalString the original String - * @param stringToConcat the string to concat - * @param stringSeparator the string separator - * @return the string + * @param fileName the file name + * @return the resources URI */ - public static String concatStringIfValueDontExist(String originalString, String stringToConcat, String stringSeparator) { - if (originalString.isEmpty() || originalString.isBlank()) { - stringSeparator = ""; + @NonNull + public static URI getResourcesURI(String fileName) { + URI overrideFile; + var resource = TestUtils.class.getClassLoader().getResource(fileName); + try { + if (resource == null) { + throw new IllegalArgumentException("Cannot find resource " + fileName + " on classpath"); + } + overrideFile = resource.toURI(); } - - if (!originalString.contains(stringToConcat)) { - originalString = originalString.concat(stringSeparator + stringToConcat); + catch (URISyntaxException e) { + throw new IllegalStateException("Cannot determine file system path for " + resource, e); } - return originalString; + return overrideFile; } } diff --git a/kroxylicious-systemtests/src/main/resources/helm_strimzi_overrides.yaml b/kroxylicious-systemtests/src/main/resources/helm_strimzi_overrides.yaml new file mode 100644 index 0000000000..b0f3ced885 --- /dev/null +++ b/kroxylicious-systemtests/src/main/resources/helm_strimzi_overrides.yaml @@ -0,0 +1,10 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +# Helm Overrides File for Strimzi used by the system tests. +replicas: 1 +#featureGates: "" + diff --git a/kroxylicious-systemtests/src/main/resources/io/kroxylicious/systemtests/installation/vault/helm_vault_overrides.yaml b/kroxylicious-systemtests/src/main/resources/helm_vault_overrides.yaml similarity index 100% rename from kroxylicious-systemtests/src/main/resources/io/kroxylicious/systemtests/installation/vault/helm_vault_overrides.yaml rename to kroxylicious-systemtests/src/main/resources/helm_vault_overrides.yaml diff --git a/kroxylicious-systemtests/src/main/resources/metadata.properties b/kroxylicious-systemtests/src/main/resources/metadata.properties index acc7fb09b8..7c727fbb71 100644 --- a/kroxylicious-systemtests/src/main/resources/metadata.properties +++ b/kroxylicious-systemtests/src/main/resources/metadata.properties @@ -6,3 +6,4 @@ kroxylicious.version=${project.version} kafka.version=${kafka.version} +strimzi.version=${strimzi.version} diff --git a/pom.xml b/pom.xml index 9c14e85913..6e1f9e62da 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ 0.8.12 - 0.40.0 + 0.41.0 6.12.1 3.4.1 3.9.6