upgraded strimzi to 0.41.0 and improved strimzi installation to be ab…

…le to select the version for system tests (kroxylicious#1211) * improved strimzi installation to be able to select the version for system tests * imports sorted * removed unused dependency in system tests pom * replaced exception * return URI instead of Path for getting resources * Update kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java Co-authored-by: Keith Wall <[email protected]> Signed-off-by: Francisco Vila <[email protected]> * commented feature gates strimzi helm file --------- Signed-off-by: Francisco Vila <[email protected]> Co-authored-by: Keith Wall <[email protected]>
franvila · May 16, 2024 · 8cc4cdb · 8cc4cdb
1 parent fb835a5
commit 8cc4cdb
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 116 deletions.
diff --git a/DEV_GUIDE.md b/DEV_GUIDE.md
@@ -336,7 +336,7 @@ has been applied ineffectively.
 * `KROXYLICIOUS_IMAGE_REPO`: url to the image of kroxylicious to be used. Default value: `quay.io/kroxylicious/kroxylicious`
 * `KROXYLICIOUS_VERSION`: version of kroxylicious to be used. Default value: `${project.version}` in pom file
 * `KAFKA_VERSION`: kafka version to be used. Default value: `${kafka.version}` in pom file
-* `STRIMZI_URL`: url where to download strimzi. Default value: `https://strimzi.io/install/latest?namespace=kafka`
+* `STRIMZI_VERSION`: strimzi version to be used. Default value: `${strimzi.version}` in pom file
 * `SKIP_TEARDOWN`: variable for development purposes to avoid keep deploying and deleting deployments each run. Default value: `false`
 * `CONTAINER_CONFIG_PATH`: directory where `config.json` file is located. This file contains the pull secrets to be used by
 the container engine. Default value: `$HOME/.docker/config.json`

diff --git a/kroxylicious-systemtests/pom.xml b/kroxylicious-systemtests/pom.xml
@@ -32,10 +32,6 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-yaml</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.fabric8</groupId>
             <artifactId>kubernetes-client-api</artifactId>

diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java
@@ -26,13 +26,13 @@ private Environment() {
     private static final String KAFKA_VERSION_ENV = "KAFKA_VERSION";
     private static final String KROXY_VERSION_ENV = "KROXYLICIOUS_VERSION";
     private static final String KROXY_IMAGE_REPO_ENV = "KROXYLICIOUS_IMAGE_REPO";
-    private static final String STRIMZI_URL_ENV = "STRIMZI_URL";
     private static final String SKIP_TEARDOWN_ENV = "SKIP_TEARDOWN";
     public static final String STRIMZI_FEATURE_GATES_ENV = "STRIMZI_FEATURE_GATES";
     private static final String CONTAINER_CONFIG_PATH_ENV = "CONTAINER_CONFIG_PATH";
     private static final String VAULT_CHART_VERSION_ENV = "VAULT_CHART_VERSION";
     private static final String SKIP_STRIMZI_INSTALL_ENV = "SKIP_STRIMZI_INSTALL";
     private static final String KAFKA_CLIENT_ENV = "KAFKA_CLIENT";
+    private static final String STRIMZI_VERSION_ENV = "STRIMZI_VERSION";
 
     /**
      * The kafka version default value
@@ -53,14 +53,19 @@ private Environment() {
     }
 
     /**
-     * The url where kroxylicious image lives to be downloaded.
+     * The kafka version default value
      */
-    private static final String KROXY_IMAGE_REPO_DEFAULT = "quay.io/kroxylicious/kroxylicious";
+    private static final String STRIMZI_VERSION_DEFAULT;
+
+    static {
+        STRIMZI_VERSION_DEFAULT = determineStrimziVersion();
+    }
 
     /**
-     * The strimzi installation url for kubernetes.
+     * The url where kroxylicious image lives to be downloaded.
      */
-    private static final String STRIMZI_URL_DEFAULT = "https://strimzi.io/install/latest?namespace=" + Constants.KAFKA_DEFAULT_NAMESPACE;
+    private static final String KROXY_IMAGE_REPO_DEFAULT = "quay.io/kroxylicious/kroxylicious";
+
     /**
      * The default value for skipping the teardown locally.
      */
@@ -80,10 +85,7 @@ private Environment() {
      * KROXY_VERSION env variable assignment
      */
     public static final String KROXY_VERSION = getOrDefault(KROXY_VERSION_ENV, KROXY_VERSION_DEFAULT);
-    /**
-     * STRIMZI_URL env variable assignment
-     */
-    public static final String STRIMZI_URL = getOrDefault(STRIMZI_URL_ENV, STRIMZI_URL_DEFAULT);
+
     /**
      * KROXY_IMAGE_REPO env variable assignment
      */
@@ -103,6 +105,8 @@ private Environment() {
 
     public static final String KAFKA_CLIENT = getOrDefault(KAFKA_CLIENT_ENV, KAFKA_CLIENT_DEFAULT);
 
+    public static final String STRIMZI_VERSION = getOrDefault(STRIMZI_VERSION_ENV, STRIMZI_VERSION_DEFAULT);
+
     private static String getOrDefault(String varName, String defaultValue) {
         return getOrDefault(varName, String::toString, defaultValue);
     }
@@ -139,4 +143,8 @@ private static String determineKroxyliciousVersion() {
     private static String determineKafkaVersion() {
         return readMetadataProperty("kafka.version");
     }
+
+    private static String determineStrimziVersion() {
+        return readMetadataProperty("strimzi.version");
+    }
 }
diff --git a/...s-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java b/...s-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java
@@ -6,30 +6,16 @@
 
 package io.kroxylicious.systemtests.installation.strimzi;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InvalidObjectException;
-import java.util.List;
+import java.nio.file.Path;
+import java.util.Optional;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLParser;
-
-import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable;
-
 import io.kroxylicious.systemtests.Constants;
 import io.kroxylicious.systemtests.Environment;
-import io.kroxylicious.systemtests.utils.DeploymentUtils;
+import io.kroxylicious.systemtests.resources.manager.ResourceManager;
 import io.kroxylicious.systemtests.utils.TestUtils;
 
 import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient;
@@ -39,60 +25,18 @@
  */
 public class Strimzi {
     private static final Logger LOGGER = LoggerFactory.getLogger(Strimzi.class);
+
+    public static final String STRIMZI_HELM_REPOSITORY = "quay.io/strimzi-helm/strimzi-kafka-operator";
+    public static final String STRIMZI_SERVICE_NAME = "strimzi-cluster-operator";
     private final String deploymentNamespace;
-    private final NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable<HasMetadata> deployment;
 
     /**
      * Instantiates a new Strimzi.
      *
      * @param deploymentNamespace the deployment namespace
      */
-    public Strimzi(String deploymentNamespace) throws IOException {
+    public Strimzi(String deploymentNamespace) {
         this.deploymentNamespace = deploymentNamespace;
-        InputStream strimziMultiDocumentYaml = DeploymentUtils.getDeploymentFileFromURL(Environment.STRIMZI_URL);
-        InputStream strimziMultiDocumentYamlKRaft = configureKRaftModeForStrimzi(strimziMultiDocumentYaml);
-        deployment = kubeClient().getClient().load(strimziMultiDocumentYamlKRaft);
-    }
-
-    private InputStream configureKRaftModeForStrimzi(InputStream strimziMultiDocumentYaml) throws IOException {
-        YAMLFactory yamlFactory = new YAMLFactory();
-        ObjectMapper mapper = new YAMLMapper();
-
-        YAMLParser multiDocumentYamlParser = yamlFactory.createParser(strimziMultiDocumentYaml);
-        List<JsonNode> docs = mapper
-                .readValues(multiDocumentYamlParser, new TypeReference<JsonNode>() {
-                })
-                .readAll();
-        boolean found = false;
-        List<JsonNode> deploymentDocs = docs.stream().filter(p -> p.at("/kind").asText("").equals(Constants.DEPLOYMENT)).toList();
-        for (JsonNode doc : deploymentDocs) {
-            ArrayNode arrayNode = (ArrayNode) doc.at("/spec/template/spec/containers");
-            ArrayNode envNode = (ArrayNode) arrayNode.get(0).at("/env");
-            for (JsonNode node : envNode) {
-                if (node.at("/name").asText().equals(Environment.STRIMZI_FEATURE_GATES_ENV)) {
-                    found = true;
-                    String value = node.at("/value").asText();
-                    if (value.isEmpty() || value.isBlank()) {
-                        value = String.join(",", Constants.USE_KRAFT_MODE, Constants.USE_KAFKA_NODE_POOLS);
-                    }
-                    else {
-                        value = value.replace(Constants.DONT_USE_KRAFT_MODE, Constants.USE_KRAFT_MODE)
-                                .replace(Constants.DONT_USE_KAFKA_NODE_POOLS, Constants.USE_KAFKA_NODE_POOLS);
-
-                        value = TestUtils.concatStringIfValueDontExist(value, Constants.USE_KRAFT_MODE, ",");
-                        value = TestUtils.concatStringIfValueDontExist(value, Constants.USE_KAFKA_NODE_POOLS, ",");
-                    }
-                    ((ObjectNode) node).put("value", value);
-                    break;
-                }
-            }
-        }
-
-        if (!found) {
-            throw new InvalidObjectException("STRIMZI_FEATURE_GATES env variable not found in yaml!");
-        }
-
-        return new ByteArrayInputStream(mapper.writeValueAsBytes(docs));
     }
 
     /**
@@ -105,8 +49,11 @@ public void deploy() {
             LOGGER.warn("Skipping strimzi deployment. It is already deployed!");
             return;
         }
-        deployment.inNamespace(deploymentNamespace).create();
-        DeploymentUtils.waitForDeploymentReady(deploymentNamespace, Constants.STRIMZI_DEPLOYMENT_NAME);
+
+        ResourceManager.helmClient().namespace(deploymentNamespace).installByContainerImage(STRIMZI_HELM_REPOSITORY, STRIMZI_SERVICE_NAME,
+                Optional.of(Environment.STRIMZI_VERSION),
+                Optional.of(Path.of(TestUtils.getResourcesURI("helm_strimzi_overrides.yaml"))),
+                Optional.empty());
     }
 
     /**
@@ -119,7 +66,6 @@ public void delete() throws IOException {
             return;
         }
         LOGGER.info("Deleting Strimzi in {} namespace", deploymentNamespace);
-        deployment.inNamespace(deploymentNamespace).delete();
-        DeploymentUtils.waitForDeploymentDeletion(deploymentNamespace, Constants.STRIMZI_DEPLOYMENT_NAME);
+        ResourceManager.helmClient().namespace(deploymentNamespace).delete(STRIMZI_SERVICE_NAME);
     }
 }
diff --git a/...cious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/vault/Vault.java b/...cious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/vault/Vault.java
@@ -9,7 +9,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.Map;
@@ -25,8 +24,7 @@
 import io.kroxylicious.systemtests.resources.manager.ResourceManager;
 import io.kroxylicious.systemtests.utils.DeploymentUtils;
 import io.kroxylicious.systemtests.utils.NamespaceUtils;
-
-import edu.umd.cs.findbugs.annotations.NonNull;
+import io.kroxylicious.systemtests.utils.TestUtils;
 
 import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient;
 
@@ -136,30 +134,13 @@ public void deploy() {
         ResourceManager.helmClient().addRepository(VAULT_HELM_REPOSITORY_NAME, VAULT_HELM_REPOSITORY_URL);
         ResourceManager.helmClient().namespace(deploymentNamespace).install(VAULT_HELM_CHART_NAME, VAULT_SERVICE_NAME,
                 Optional.of(Environment.VAULT_CHART_VERSION),
-                Optional.of(getHelmOverridePath()),
+                Optional.of(Path.of(TestUtils.getResourcesURI("helm_vault_overrides.yaml"))),
                 Optional.of(Map.of("server.dev.devRootToken", vaultRootToken,
                         "global.openshift", String.valueOf(openshiftCluster))));
 
         DeploymentUtils.waitForDeploymentRunning(deploymentNamespace, VAULT_POD_NAME, Duration.ofMinutes(1));
     }
 
-    @NonNull
-    private Path getHelmOverridePath() {
-        var name = "helm_vault_overrides.yaml";
-        Path overrideFile;
-        var resource = getClass().getResource(name);
-        try {
-            if (resource == null) {
-                throw new IllegalStateException("Cannot find override resource " + name + " on classpath");
-            }
-            overrideFile = Path.of(resource.toURI());
-        }
-        catch (URISyntaxException e) {
-            throw new IllegalStateException("Cannot determine file system path for " + resource);
-        }
-        return overrideFile;
-    }
-
     /**
      * Delete.
      *

diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/HelmClient.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/HelmClient.java
@@ -98,6 +98,11 @@ public HelmClient install(String chartName, String releaseName, Optional<String>
         return this;
     }
 
+    public HelmClient installByContainerImage(String helmRepositoryOci, String releaseName, Optional<String> version, Optional<Path> overrideFile,
+                                              Optional<Map<String, String>> overrideMap) {
+        return install("oci://" + helmRepositoryOci, releaseName, version, overrideFile, overrideMap);
+    }
+
     /**
      * Add repository to the helm client.
      *
@@ -119,7 +124,7 @@ public HelmClient addRepository(String repoName, String repoUrl) {
      */
     public HelmClient delete(String releaseName) {
         LOGGER.info("Deleting helm-chart {}", releaseName);
-        delete(String.valueOf(namespace), releaseName);
+        delete(namespace.orElse("default"), releaseName);
         return this;
     }
 

diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java
@@ -6,16 +6,23 @@
 
 package io.kroxylicious.systemtests.utils;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.Set;
 
+import edu.umd.cs.findbugs.annotations.NonNull;
+
 /**
  * The type Test utils.
  */
 public class TestUtils {
 
+    private TestUtils() {
+    }
+
     /**
      * Gets default posix file permissions.
      *
@@ -26,21 +33,24 @@ public static FileAttribute<Set<PosixFilePermission>> getDefaultPosixFilePermiss
     }
 
     /**
-     * Concat string if value don't exist.
+     * Gets resources URI.
      *
-     * @param originalString the original String
-     * @param stringToConcat the string to concat
-     * @param stringSeparator the string separator
-     * @return the string
+     * @param fileName the file name
+     * @return the resources URI
      */
-    public static String concatStringIfValueDontExist(String originalString, String stringToConcat, String stringSeparator) {
-        if (originalString.isEmpty() || originalString.isBlank()) {
-            stringSeparator = "";
+    @NonNull
+    public static URI getResourcesURI(String fileName) {
+        URI overrideFile;
+        var resource = TestUtils.class.getClassLoader().getResource(fileName);
+        try {
+            if (resource == null) {
+                throw new IllegalArgumentException("Cannot find resource " + fileName + " on classpath");
+            }
+            overrideFile = resource.toURI();
         }
-
-        if (!originalString.contains(stringToConcat)) {
-            originalString = originalString.concat(stringSeparator + stringToConcat);
+        catch (URISyntaxException e) {
+            throw new IllegalStateException("Cannot determine file system path for " + resource, e);
         }
-        return originalString;
+        return overrideFile;
     }
 }
diff --git a/kroxylicious-systemtests/src/main/resources/helm_strimzi_overrides.yaml b/kroxylicious-systemtests/src/main/resources/helm_strimzi_overrides.yaml
@@ -0,0 +1,10 @@
+#
+# Copyright Kroxylicious Authors.
+#
+# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
+#
+
+# Helm Overrides File for Strimzi used by the system tests.
+replicas: 1
+#featureGates: ""
+
diff --git a/...tallation/vault/helm_vault_overrides.yaml → .../main/resources/helm_vault_overrides.yaml b/...tallation/vault/helm_vault_overrides.yaml → .../main/resources/helm_vault_overrides.yaml
diff --git a/kroxylicious-systemtests/src/main/resources/metadata.properties b/kroxylicious-systemtests/src/main/resources/metadata.properties
@@ -6,3 +6,4 @@
 
 kroxylicious.version=${project.version}
 kafka.version=${kafka.version}
+strimzi.version=${strimzi.version}
diff --git a/pom.xml b/pom.xml
@@ -61,7 +61,7 @@
         <jacoco-maven-plugin.version>0.8.12</jacoco-maven-plugin.version>
 
         <!-- Test dependencies -->
-        <strimzi.version>0.40.0</strimzi.version>
+        <strimzi.version>0.41.0</strimzi.version>
         <kubernetes-client.version>6.12.1</kubernetes-client.version>
         <enforcer.api.version>3.4.1</enforcer.api.version>
         <maven.core.version>3.9.6</maven.core.version>
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,3 +6,4 @@

		kroxylicious.version=${project.version}
		kafka.version=${kafka.version}
		strimzi.version=${strimzi.version}