diff --git a/.gitignore b/.gitignore index 99cbd6dc8e..956680c94b 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,9 @@ out/ # mpeltonen/sbt-idea plugin .idea_modules/ + # Javadoc plugin +.idea/**/intellij-javadocs-*.xml + # JIRA plugin atlassian-ide-plugin.xml diff --git a/DEV_GUIDE.md b/DEV_GUIDE.md index 2da866d429..98453488be 100644 --- a/DEV_GUIDE.md +++ b/DEV_GUIDE.md @@ -68,6 +68,7 @@ The running of the tests can be controlled with the following Maven properties: |--------------------|-------------------------------------------------------------------------------------------| | `-DskipUTs=true` | skip unit tests | | `-DskipITs=true` | skip integration tests | +| `-DskipSTs=true` | skip system tests | | `-DskipTests=true` | skip all tests | | `-Pdebug` | enables logging so you can see what the Kafka clients, Proxy and in VM brokers are up to. | @@ -308,6 +309,50 @@ You'll see an API response. If the service_timeout change is effective, the soc will continue for 3 minutes. If `socat` terminates after about 10 seconds, the workaround has been applied ineffectively. +## Running system tests locally +### Prerequisites +* minikube +* User must have access to a container registry such as [quay.io](https://quay.io) or [docker.io](https://docker.io). + Create a public accessible repository within the registry named `kroxylicious`. + +### Environment variables +* `KROXYLICIOUS_IMAGE_REPO`: url to the image of kroxylicious to be used. Default value: `quay.io/kroxylicious/kroxylicious-developer` +* `KROXYLICIOUS_VERSION`: version of kroxylicious to be used. Default value: `0.4.0-SNAPSHOT` +* `KAFKA_VERSION`: kafka version to be used. Default value: `3.6.0` +* `STRIMZI_URL`: url where to download strimzi. Default value: `khttps://strimzi.io/install/latest?namespace=kafka` + +### Launch system tests +First of all, the code must be compiled and the distribution artifacts created: + +```shell +mvn clean install -Dquick -Pdist +``` + +If the tests are going to be run against local changes, +upload your package to the `kroxylicious` repository in the container registry: + +```shell +PUSH_IMAGE=true REGISTRY_DESTINATION=//kroxylicious ./scripts/deploy-image.sh +``` + +Start minikube: +```shell +minikube start +``` + +Then, you can run them from system test or root folder: + +* Run the system tests from [kroxylicious-systemtests](kroxylicious-systemtests) folder: + +```shell +KROXYLICIOUS_IMAGE_REPO=//kroxylicious mvn clean integration-test +``` + +* Run them from root folder of kroxylicious project: + +```shell +KROXYLICIOUS_IMAGE_REPO=//kroxylicious mvn clean verify -DskiptITs=true -DskiptUTs=true -DskipSTs=false +``` ## Rendering documentation diff --git a/kroxylicious-systemtests/pom.xml b/kroxylicious-systemtests/pom.xml new file mode 100644 index 0000000000..9781e39a74 --- /dev/null +++ b/kroxylicious-systemtests/pom.xml @@ -0,0 +1,115 @@ + + + + + 4.0.0 + + io.kroxylicious + kroxylicious-parent + 0.4.0-SNAPSHOT + + + kroxylicious-systemtests + + The intention of this module is to test Kroxylicious against a kafka instance deployed in a kubernetes cluster/minikube + using Strimzi, to simulate a real scenario for end-to-end testing treating Kroxylicious as a black box. + + + + io.fabric8 + kubernetes-client-api + + + io.fabric8 + kubernetes-client + runtime + + + io.fabric8 + kubernetes-httpclient-jdk + runtime + + + org.junit.jupiter + junit-jupiter-engine + test + + + io.strimzi + api + + + org.junit.jupiter + junit-jupiter-api + compile + + + + org.slf4j + slf4j-api + + + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + org.apache.maven.enforcer + enforcer-api + + + org.apache.maven + maven-core + + + org.hamcrest + hamcrest + + + org.junit.jupiter + junit-jupiter-params + + + org.awaitility + awaitility + + + commons-io + commons-io + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + integration-test + + test + + integration-test + + ${skipSTs} + + **/ST*.java + **/*ST.java + + + + + + + + diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java new file mode 100644 index 0000000000..bb6d65f5d6 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java @@ -0,0 +1,142 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests; + +import java.time.Duration; + +import static io.kroxylicious.systemtests.Environment.KAFKA_VERSION_DEFAULT; + +/** + * The interface Constants. + */ +public interface Constants { + + /** + * The deployment name for kroxylicous + */ + String KROXY_DEPLOYMENT_NAME = "kroxylicious-proxy"; + /** + * The service name for kroxylicious. Used for the bootstrap url + */ + String KROXY_SERVICE_NAME = "kroxylicious-service"; + /** + * The constant KROXY_CONFIG_NAME. + */ + String KROXY_CONFIG_NAME = "kroxylicious-config"; + /** + * Strimzi cluster operator deployment name + */ + String STRIMZI_DEPLOYMENT_NAME = "strimzi-cluster-operator"; + /** + * The default namespace used for kubernetes deployment + */ + String KROXY_DEFAULT_NAMESPACE = "kafka"; + + /** + * The cert-manager namespace for kubernetes deployment + */ + String CERT_MANAGER_NAMESPACE = "cert-manager"; + + /** + * API versions of Strimzi CustomResources + */ + String KAFKA_API_VERSION_V1BETA2 = "kafka.strimzi.io/v1beta2"; + + /** + * Kind of Strimzi CustomResources + */ + String KAFKA_KIND = "Kafka"; + /** + * Kind of kafka topics + */ + String KAFKA_TOPIC_KIND = "KafkaTopic"; + /** + * Kind of kafka users + */ + String KAFKA_USER_KIND = "KafkaUser"; + /** + * Kind of kafka node pools + */ + String KAFKA_NODE_POOL_KIND = "KafkaNodePool"; + /** + * Kind of pods + */ + String POD_KIND = "Pod"; + + /** + * Kind of config maps + */ + String CONFIG_MAP_KIND = "ConfigMap"; + + /** + * Kind of services + */ + String SERVICE_KIND = "Service"; + + /** + * Load balancer type name. + */ + String LOAD_BALANCER_TYPE = "LoadBalancer"; + + /** + * Listener names for Kafka cluster + */ + String PLAIN_LISTENER_NAME = "plain"; + /** + * Listener name for tls + */ + String TLS_LISTENER_NAME = "tls"; + + /** + * Strimzi related labels and annotations + */ + String STRIMZI_DOMAIN = "strimzi.io/"; + /** + * Strimzi cluster label + */ + String STRIMZI_CLUSTER_LABEL = STRIMZI_DOMAIN + "cluster"; + + /** + * Polls and timeouts constants + */ + long POLL_INTERVAL_FOR_RESOURCE_READINESS_MILLIS = Duration.ofSeconds(5).toMillis(); + /** + * Poll interval for resource deletion in milliseconds + */ + long POLL_INTERVAL_FOR_RESOURCE_DELETION_MILLIS = Duration.ofSeconds(1).toMillis(); + + /** + * Global timeout in milliseconds + */ + long GLOBAL_TIMEOUT_MILLIS = Duration.ofMinutes(5).toMillis(); + /** + * Global Poll interval in milliseconds + */ + long GLOBAL_POLL_INTERVAL_MILLIS = Duration.ofSeconds(1).toMillis(); + + /** + * Kubernetes related constants + */ + String DEPLOYMENT = "Deployment"; + /** + * Strimzi kafka image url in quay + */ + String STRIMZI_KAFKA_IMAGE = "quay.io/strimzi/kafka:latest-kafka-" + KAFKA_VERSION_DEFAULT; + + /** + * The cert manager url to install it on kubernetes + */ + String CERT_MANAGER_URL = "https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml"; + /** + * kafka consumer client label to identify the consumer test client + */ + String KAFKA_CONSUMER_CLIENT_LABEL = "kafka-consumer-client"; + /** + * kafka producer client label to identify the producer test client + */ + String KAFKA_PRODUCER_CLIENT_LABEL = "kafka-producer-client"; +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java new file mode 100644 index 0000000000..8487d92563 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java @@ -0,0 +1,71 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests; + +import java.util.function.Function; + +/** + * The type Environment. + */ +public class Environment { + + private Environment() { + } + + /** + * Env. variables names + */ + private static final String KAFKA_VERSION_ENV = "KAFKA_VERSION"; + private static final String KROXY_VERSION_ENV = "KROXYLICIOUS_VERSION"; + private static final String KROXY_IMAGE_REPO_ENV = "KROXYLICIOUS_IMAGE_REPO"; + private static final String STRIMZI_URL_ENV = "STRIMZI_URL"; + + /** + * The kafka version default value + */ + public static final String KAFKA_VERSION_DEFAULT = "3.6.0"; + + /** + * The kroxy version default value + */ + public static final String KROXY_VERSION_DEFAULT = "0.4.0-SNAPSHOT"; + /** + * The url where kroxylicious image lives to be downloaded. + */ + public static final String KROXY_IMAGE_REPO_DEFAULT = "quay.io/kroxylicious/kroxylicious-developer"; + + /** + * The strimzi installation url for kubernetes. + */ + public static final String STRIMZI_URL_DEFAULT = "https://strimzi.io/install/latest?namespace=" + Constants.KROXY_DEFAULT_NAMESPACE; + + /** + * KAFKA_VERSION env variable assignment + */ + public static final String KAFKA_VERSION = getOrDefault(KAFKA_VERSION_ENV, KAFKA_VERSION_DEFAULT); + + /** + * KROXY_VERSION env variable assignment + */ + public static final String KROXY_VERSION = getOrDefault(KROXY_VERSION_ENV, KROXY_VERSION_DEFAULT); + /** + * STRIMZI_URL env variable assignment + */ + public static final String STRIMZI_URL = getOrDefault(STRIMZI_URL_ENV, STRIMZI_URL_DEFAULT); + /** + * KROXY_IMAGE_REPO env variable assignment + */ + public static final String KROXY_IMAGE_REPO = getOrDefault(KROXY_IMAGE_REPO_ENV, KROXY_IMAGE_REPO_DEFAULT); + + private static String getOrDefault(String varName, String defaultValue) { + return getOrDefault(varName, String::toString, defaultValue); + } + + private static T getOrDefault(String varName, Function converter, T defaultValue) { + return System.getenv(varName) != null ? converter.apply(System.getenv(varName)) : defaultValue; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/enums/ConditionStatus.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/enums/ConditionStatus.java new file mode 100644 index 0000000000..38bd69746a --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/enums/ConditionStatus.java @@ -0,0 +1,21 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.enums; + +/** + * Status of the CR, found inside .status.conditions.*.status + */ +public enum ConditionStatus { + /** + *True condition status. + */ + TRUE, + /** + *False condition status. + */ + FALSE +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/enums/LogLevel.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/enums/LogLevel.java new file mode 100644 index 0000000000..ea8ffabe1a --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/enums/LogLevel.java @@ -0,0 +1,29 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.enums; + +/** + * The enum Log level. + */ +public enum LogLevel { + /** + *Debug log level. + */ + DEBUG, + /** + *Trace log level. + */ + TRACE, + /** + *Info log level. + */ + INFO, + /** + *Error log level. + */ + ERROR +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/executor/Exec.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/executor/Exec.java new file mode 100644 index 0000000000..28c60bcd8a --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/executor/Exec.java @@ -0,0 +1,480 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.executor; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.k8s.exception.KubeClusterException; + +import static java.lang.String.join; + +/** + * Class provide execution of external command + */ +@SuppressWarnings({ "checkstyle:ClassDataAbstractionCoupling", "checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity" }) +public class Exec { + private static final Logger LOGGER = LoggerFactory.getLogger(Exec.class); + private static final Pattern ERROR_PATTERN = Pattern.compile("Error from server \\(([a-zA-Z0-9]+)\\):"); + private static final Pattern INVALID_PATTERN = Pattern.compile("The ([a-zA-Z0-9]+) \"([a-z0-9.-]+)\" is invalid:"); + private static final Pattern PATH_SPLITTER = Pattern.compile(System.getProperty("path.separator")); + private static final int MAXIMUM_EXEC_LOG_CHARACTER_SIZE = Integer.parseInt(System.getenv().getOrDefault("STRIMZI_EXEC_MAX_LOG_OUTPUT_CHARACTERS", "20000")); + private static final Object LOCK = new Object(); + + /** + * The Process. + */ + private Process process; + private String stdOut; + private String stdErr; + private StreamGobbler stdOutReader; + private StreamGobbler stdErrReader; + private Path logPath; + + /** + * Instantiates a new Exec. + */ + public Exec() { + } + + /** + * Getter for stdOutput + * + * @return string stdOut + */ + public String out() { + return stdOut; + } + + /** + * Getter for stdErrorOutput + * + * @return string stdErr + */ + public String err() { + return stdErr; + } + + /** + * Is running boolean. + * + * @return the boolean + */ + public boolean isRunning() { + return process.isAlive(); + } + + /** + * Gets ret code. + * + * @return the ret code + */ + public int getRetCode() { + LOGGER.info("Process: {}", process); + if (isRunning()) { + return -1; + } + else { + return process.exitValue(); + } + } + + /** + * Method executes external command + * + * @param dir the dir + * @param command arguments for command + * @return execution results + */ + public static ExecResult exec(File dir, String... command) { + return exec(Arrays.asList(command), dir); + } + + /** + * Exec exec result. + * + * @param command the command + * @return the exec result + */ + public static ExecResult exec(String... command) { + return exec(Arrays.asList(command), null); + } + + /** + * Exec without wait. + * + * @param command the command + * @return the pid + */ + public static long execWithoutWait(String... command) { + return execWithoutWait(Arrays.asList(command)); + } + + /** + * Method executes external command + * + * @param command arguments for command + * @param dir the dir + * @return execution results + */ + public static ExecResult exec(List command, File dir) { + return exec(null, command, 0, false, dir); + } + + /** + * Exec exec result. + * + * @param command the command + * @return the exec result + */ + public static ExecResult exec(List command) { + return exec(null, command, 0, false); + } + + /** + * Method executes external command + * + * @param input the input + * @param command arguments for command + * @return execution results + */ + public static ExecResult exec(String input, List command) { + return exec(input, command, 0, false); + } + + /** + * Method executes external command + * @param input the input + * @param command arguments for command + * @param timeout timeout for execution + * @param logToOutput log output or not + * @param dir the dir + * @return execution results + */ + public static ExecResult exec(String input, List command, int timeout, boolean logToOutput, File dir) { + return exec(input, command, timeout, logToOutput, true, dir); + } + + /** + * Exec without wait exec result. + * + * @param command the command + * @return the pid + */ + public static long execWithoutWait(List command) { + Exec executor = new Exec(); + return executor.executeWithoutWait(command, null); + } + + /** + * Exec exec result. + * + * @param input the input + * @param command the command + * @param timeout the timeout + * @param logToOutput the log to output + * @return the exec result + */ + public static ExecResult exec(String input, List command, int timeout, boolean logToOutput) { + return exec(input, command, timeout, logToOutput, true, null); + } + + /** + * Method executes external command + * @param input the input + * @param command arguments for command + * @param timeout timeout for execution + * @param logToOutput log output or not + * @param throwErrors look for errors in output and throws exception if true + * @param dir the dir + * @return execution results + */ + public static ExecResult exec(String input, List command, int timeout, boolean logToOutput, boolean throwErrors, File dir) { + int ret; + ExecResult execResult; + try { + Exec executor = new Exec(); + ret = executor.execute(input, command, timeout, dir); + synchronized (LOCK) { + if (logToOutput || ret != 0) { + logExecutor(ret, input, command, executor.out(), executor.err()); + } + } + + execResult = new ExecResult(ret, executor.out(), executor.err()); + + if (throwErrors && ret != 0) { + String msg = "`" + join(" ", command) + "` got status code " + ret + " and stderr:\n------\n" + executor.stdErr + "\n------\nand stdout:\n------\n" + + executor.stdOut + "\n------"; + + throwExceptionForErrorPattern(msg, executor.err(), execResult); + } + return execResult; + + } + catch (IOException | ExecutionException e) { + throw new KubeClusterException(e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new KubeClusterException(e); + } + } + + private static void throwExceptionForErrorPattern(String msg, String err, ExecResult execResult) { + Matcher matcher = ERROR_PATTERN.matcher(err); + KubeClusterException kubeClusterException = new KubeClusterException(execResult, msg); + + if (matcher.find()) { + switch (matcher.group(1)) { + case "NotFound": + kubeClusterException = new KubeClusterException.NotFound(execResult, msg); + break; + case "AlreadyExists": + kubeClusterException = new KubeClusterException.AlreadyExists(execResult, msg); + break; + default: + break; + } + } + matcher = INVALID_PATTERN.matcher(err); + if (matcher.find()) { + kubeClusterException = new KubeClusterException.InvalidResource(execResult, msg); + } + throw kubeClusterException; + } + + private static void logExecutor(int ret, String input, List command, String execOut, String execErr) { + String log = ret != 0 ? "Failed to exec command" : "Command"; + String commandLine = command == null || command.isEmpty() ? "" : String.join(" ", command); + LOGGER.info("{}: {}", log, commandLine); + if (input != null && !input.contains("CustomResourceDefinition")) { + LOGGER.info("Input: {}", input); + } + LOGGER.info("RETURN code: {}", ret); + if (!execOut.isEmpty()) { + LOGGER.debug("======STDOUT START======="); + LOGGER.debug("{}", cutExecutorLog(execOut)); + LOGGER.debug("======STDOUT END======"); + } + if (!execErr.isEmpty()) { + LOGGER.debug("======STDERR START======="); + LOGGER.debug("{}", cutExecutorLog(execErr)); + LOGGER.debug("======STDERR END======"); + } + } + + /** + * Method executes external command + * + * @param input the input + * @param commands arguments for command + * @param timeoutMs timeout in ms for kill + * @param dir the dir + * @return returns ecode of execution + * @throws IOException the io exception + * @throws InterruptedException the interrupted exception + * @throws ExecutionException the execution exception + */ + public int execute(String input, List commands, long timeoutMs, File dir) throws IOException, InterruptedException, ExecutionException { + LOGGER.debug("Running command - {}", String.join(" ", commands.toArray(new String[0]))); + ProcessBuilder builder = new ProcessBuilder(); + builder.command(commands); + dir = dir == null ? new File(System.getProperty("user.dir")) : dir; + builder.directory(dir); + process = builder.start(); + OutputStream outputStream = process.getOutputStream(); + if (input != null) { + LOGGER.debug("With stdin {}", input); + outputStream.write(input.getBytes(Charset.defaultCharset())); + } + // Close subprocess' stdin + outputStream.close(); + + Future output = readStdOutput(); + Future error = readStdError(); + + int retCode = 1; + if (timeoutMs > 0) { + if (process.waitFor(timeoutMs, TimeUnit.MILLISECONDS)) { + retCode = process.exitValue(); + } + else { + process.destroyForcibly(); + } + } + else { + retCode = process.waitFor(); + } + + try { + stdOut = output.get(500, TimeUnit.MILLISECONDS); + } + catch (TimeoutException ex) { + output.cancel(true); + stdOut = stdOutReader.getData(); + } + + try { + stdErr = error.get(500, TimeUnit.MILLISECONDS); + } + catch (TimeoutException ex) { + error.cancel(true); + stdErr = stdErrReader.getData(); + } + storeOutputsToFile(); + + return retCode; + } + + /** + * Execute without waiting for response. + * + * @param commands the commands + * @param dir the dir + * @return the pid + */ + public long executeWithoutWait(List commands, File dir) { + LOGGER.debug("Running command - {}", String.join(" ", commands.toArray(new String[0]))); + ProcessBuilder builder = new ProcessBuilder(); + builder.command(commands); + dir = dir == null ? new File(System.getProperty("user.dir")) : dir; + builder.directory(dir); + try { + process = builder.start(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return process.pid(); + } + + /** + * Get standard output of execution + * + * @return future string output + */ + private Future readStdOutput() { + stdOutReader = new StreamGobbler(process.getInputStream()); + return stdOutReader.read(); + } + + /** + * Get standard error output of execution + * + * @return future string error output + */ + private Future readStdError() { + stdErrReader = new StreamGobbler(process.getErrorStream()); + return stdErrReader.read(); + } + + /** + * Get stdOut and stdErr and store it into files + */ + private void storeOutputsToFile() { + if (logPath != null) { + try { + Files.createDirectories(logPath); + Files.writeString(Paths.get(logPath.toString(), "stdOutput.log"), stdOut, Charset.defaultCharset()); + Files.writeString(Paths.get(logPath.toString(), "stdError.log"), stdErr, Charset.defaultCharset()); + } + catch (Exception ex) { + LOGGER.warn("Cannot save output of execution: {}", ex.getMessage()); + } + } + } + + /** + * Check if command is executable + * @param cmd command + * @return true.false boolean + */ + public static boolean isExecutableOnPath(String cmd) { + for (String dir : PATH_SPLITTER.split(System.getenv("PATH"))) { + if (new File(dir, cmd).canExecute()) { + return true; + } + } + return false; + } + + /** + * This method check the size of executor output log and cut it if it's too long. + * @param log executor log + * @return updated log if size is too big + */ + public static String cutExecutorLog(String log) { + if (log.trim().length() > MAXIMUM_EXEC_LOG_CHARACTER_SIZE) { + LOGGER.warn("Executor log is too long. Going to strip it and print only first {} characters", MAXIMUM_EXEC_LOG_CHARACTER_SIZE); + return log.trim().substring(0, MAXIMUM_EXEC_LOG_CHARACTER_SIZE); + } + return log.trim(); + } + + /** + * Class represent async reader + */ + class StreamGobbler { + private final InputStream is; + private final StringBuilder data = new StringBuilder(); + + /** + * Constructor of StreamGobbler + * + * @param is input stream for reading + */ + StreamGobbler(InputStream is) { + this.is = is; + } + + /** + * Return data from stream sync + * + * @return string of data + */ + public String getData() { + return data.toString(); + } + + /** + * read method + * + * @return return future string of output + */ + public Future read() { + return CompletableFuture.supplyAsync(() -> { + try { + return new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + }, runnable -> new Thread(runnable).start()); + } + } +} \ No newline at end of file diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/executor/ExecResult.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/executor/ExecResult.java new file mode 100644 index 0000000000..7aeb2776ed --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/executor/ExecResult.java @@ -0,0 +1,68 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.executor; + +import java.io.Serializable; + +/** + * The type Exec result. + * It is serializable because it is used in KubeClusterException that is serializable + */ +public class ExecResult implements Serializable { + private final int returnCode; + private final String stdOut; + private final String stdErr; + + /** + * Instantiates a new Exec result. + * + * @param returnCode the return code + * @param stdOut the std out + * @param stdErr the std err + */ + public ExecResult(int returnCode, String stdOut, String stdErr) { + this.returnCode = returnCode; + this.stdOut = stdOut; + this.stdErr = stdErr; + } + + /** + * Return true if the result is success, false otherwise. + * + * @return the boolean + */ + public boolean isSuccess() { + return returnCode == 0; + } + + /** + * Return code int. + * + * @return the int + */ + public int returnCode() { + return returnCode; + } + + /** + * Returns the stdOut string. + * + * @return the string + */ + public String out() { + return stdOut; + } + + /** + * Returns the stdErr string. + * + * @return the string + */ + public String err() { + return stdErr; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java new file mode 100644 index 0000000000..555ad2a825 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -0,0 +1,57 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.installation.kroxylicious; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.utils.DeploymentUtils; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Cert manager. + */ +public class CertManager { + private static final Logger LOGGER = LoggerFactory.getLogger(CertManager.class); + + private final NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable deployment; + + /** + * Instantiates a new Cert manager. + * + * @throws IOException the io exception + */ + public CertManager() throws IOException { + deployment = kubeClient().getClient() + .load(DeploymentUtils.getDeploymentFileFromURL(Constants.CERT_MANAGER_URL)); + } + + /** + * Deploy cert manager. + */ + public void deploy() { + LOGGER.info("Deploy cert manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); + deployment.create(); + DeploymentUtils.waitForDeploymentReady(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); + } + + /** + * Delete cert manager. + * @throws IOException the io exception + */ + public void delete() throws IOException { + LOGGER.info("Deleting Cert Manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); + deployment.withGracePeriod(0).delete(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java new file mode 100644 index 0000000000..ab7578c0b4 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java @@ -0,0 +1,78 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.installation.kroxylicious; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.Environment; +import io.kroxylicious.systemtests.k8s.exception.KubeClusterException; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; +import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousConfigTemplates; +import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousDeploymentTemplates; +import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousServiceTemplates; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kroxylicious. + */ +public class Kroxylicious { + private static final Logger LOGGER = LoggerFactory.getLogger(Kroxylicious.class); + private final String deploymentNamespace; + private final String containerImage; + private final ResourceManager resourceManager = ResourceManager.getInstance(); + + /** + * Instantiates a new KroxyliciousService to be used in kubernetes. + * + * @param deploymentNamespace the deployment namespace + */ + public Kroxylicious(String deploymentNamespace) { + this.deploymentNamespace = deploymentNamespace; + String kroxyUrl = Environment.KROXY_IMAGE_REPO + (Environment.KROXY_IMAGE_REPO.endsWith(":") ? "" : ":"); + this.containerImage = kroxyUrl + Environment.KROXY_VERSION; + } + + /** + * Deploy - Port per broker plain config + * @param clusterName the cluster name + * @param replicas the replicas + */ + public void deployPortPerBrokerPlain(String clusterName, int replicas) { + LOGGER.info("Deploy Kroxylicious in {} namespace", deploymentNamespace); + resourceManager.createResourceWithWait(KroxyliciousConfigTemplates.defaultKroxyConfig(clusterName, deploymentNamespace).build()); + resourceManager.createResourceWithWait(KroxyliciousDeploymentTemplates.defaultKroxyDeployment(deploymentNamespace, containerImage, replicas).build()); + resourceManager.createResourceWithoutWait(KroxyliciousServiceTemplates.defaultKroxyService(deploymentNamespace).build()); + } + + /** + * Gets number of replicas. + * + * @return the number of replicas + */ + public int getNumberOfReplicas() { + LOGGER.info("Getting number of replicas.."); + return kubeClient().getDeployment(deploymentNamespace, Constants.KROXY_DEPLOYMENT_NAME).getStatus().getReplicas(); + } + + /** + * Get bootstrap string. + * + * @return the bootstrap + */ + public String getBootstrap() { + String clusterIP = kubeClient().getService(deploymentNamespace, Constants.KROXY_SERVICE_NAME).getSpec().getClusterIP(); + if (clusterIP == null || clusterIP.isEmpty()) { + throw new KubeClusterException(new Throwable("Unable to get the clusterIP of Kroxylicious")); + } + String bootstrap = clusterIP + ":9292"; + LOGGER.debug("Kroxylicious bootstrap: {}", bootstrap); + return bootstrap; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousApp.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousApp.java new file mode 100644 index 0000000000..d07f2b3395 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousApp.java @@ -0,0 +1,107 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.installation.kroxylicious; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.executor.Exec; +import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousConfigTemplates; +import io.kroxylicious.systemtests.utils.TestUtils; + +import static org.awaitility.Awaitility.await; + +/** + * The type Kroxylicious app. + */ +public class KroxyliciousApp implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(KroxyliciousApp.class); + private final String clusterIp; + private final Thread thread; + private long pid; + + /** + * Instantiates a new Kroxylicious app. + * + * @param clusterIp the cluster ip + */ + public KroxyliciousApp(String clusterIp) { + this.clusterIp = clusterIp; + thread = new Thread(this, "kroxy"); + thread.start(); + } + + public void run() { + LOGGER.info("Launching kroxylicious app"); + Path parentPath = Path.of(System.getProperty("user.dir")).getParent(); + final Path targetPath = parentPath.resolve("kroxylicious-app").resolve("target"); + + final Path startScript = resolveStartScript(targetPath); + final Path configFile = generateKroxyliciousConfiguration(); + pid = Exec.execWithoutWait(startScript.toAbsolutePath().toString(), "-c", configFile.toAbsolutePath().toString()); + } + + private Path generateKroxyliciousConfiguration() { + try { + File configFile = Files.createTempFile("config", ".yaml", TestUtils.getDefaultPosixFilePermissions()).toFile(); + Files.writeString(configFile.toPath(), KroxyliciousConfigTemplates.getDefaultExternalKroxyConfigMap(clusterIp)); + configFile.deleteOnExit(); + return configFile.toPath(); + } + catch (IOException e) { + throw new IllegalStateException("Unable to generate kroxylicious configuration file", e); + } + } + + private static Path resolveStartScript(Path targetPath) { + try (Stream walkStream = Files.walk(targetPath)) { + final Optional startScript = walkStream.filter(Files::isRegularFile).filter(f -> f.endsWith("kroxylicious-start.sh")).findFirst(); + if (startScript.isEmpty()) { + throw new IllegalStateException("unable to find kroxylicious-start.sh"); + } + else { + return startScript.get(); + } + } + catch (IOException e) { + throw new IllegalStateException("unable to find kroxylicious-start.sh", e); + } + } + + /** + * Check if Kroxylicious process is running. + * + * @return the boolean + */ + public boolean isRunning() { + return ProcessHandle.of(pid).isPresent(); + } + + /** + * Wait for kroxylicious process. + */ + public void waitForKroxyliciousProcess() { + await().atMost(5, TimeUnit.SECONDS).until(() -> ProcessHandle.of(pid).isPresent()); + } + + /** + * Stop. + */ + public void stop() { + LOGGER.info("Stopping kroxylicious"); + thread.interrupt(); + ProcessHandle.of(pid).ifPresent(ProcessHandle::destroy); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java new file mode 100644 index 0000000000..077020ffd8 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/strimzi/Strimzi.java @@ -0,0 +1,60 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.installation.strimzi; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.Environment; +import io.kroxylicious.systemtests.utils.DeploymentUtils; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Strimzi. + */ +public class Strimzi { + private static final Logger LOGGER = LoggerFactory.getLogger(Strimzi.class); + private final String deploymentNamespace; + private final NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable deployment; + + /** + * Instantiates a new Strimzi. + * + * @param deploymentNamespace the deployment namespace + */ + public Strimzi(String deploymentNamespace) throws IOException { + this.deploymentNamespace = deploymentNamespace; + deployment = kubeClient().getClient().load(DeploymentUtils.getDeploymentFileFromURL(Environment.STRIMZI_URL)); + } + + /** + * Deploy strimzi. + * @throws IOException the io exception + */ + public void deploy() throws IOException { + LOGGER.info("Deploy Strimzi in {} namespace", deploymentNamespace); + deployment.inNamespace(deploymentNamespace).create(); + DeploymentUtils.waitForDeploymentReady(deploymentNamespace, Constants.STRIMZI_DEPLOYMENT_NAME); + } + + /** + * Delete strimzi. + * @throws IOException the io exception + */ + public void delete() throws IOException { + LOGGER.info("Deleting Strimzi in {} namespace", deploymentNamespace); + deployment.inNamespace(deploymentNamespace).delete(); + DeploymentUtils.waitForDeploymentDeletion(deploymentNamespace, Constants.STRIMZI_DEPLOYMENT_NAME); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/KubeClient.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/KubeClient.java new file mode 100644 index 0000000000..42fde779fa --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/KubeClient.java @@ -0,0 +1,222 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s; + +import java.util.List; +import java.util.stream.Collectors; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.client.KubernetesClient; + +/** + * The type Kube client. + */ +public class KubeClient { + + /** + * The Client. + */ + protected final KubernetesClient client; + /** + * The Namespace. + */ + protected String namespace; + + /** + * Instantiates a new Kube client. + * + * @param client the client + * @param namespace the namespace + */ + public KubeClient(KubernetesClient client, String namespace) { + this.client = client; + this.namespace = namespace; + } + + // ============================ + // ---------> CLIENT <--------- + // ============================ + + /** + * Gets client. + * + * @return the client + */ + public KubernetesClient getClient() { + return client; + } + + // =============================== + // ---------> NAMESPACE <--------- + // =============================== + + /** + * Namespace kube client. + * + * @param futureNamespace the future namespace + * @return the kube client + */ + public KubeClient namespace(String futureNamespace) { + return new KubeClient(this.client, futureNamespace); + } + + /** + * Gets namespace. + * + * @return the namespace + */ + public String getNamespace() { + return namespace; + } + + /** + * Gets namespace. + * + * @param namespace the namespace + * @return the namespace + */ + public Namespace getNamespace(String namespace) { + return client.namespaces().withName(namespace).get(); + } + + /** + * Create namespace. + * + * @param namespaceName the namespace name + */ + public void createNamespace(String namespaceName) { + Namespace ns = new NamespaceBuilder().withNewMetadata().withName(namespaceName).endMetadata().build(); + client.namespaces().resource(ns).create(); + } + + /** + * Delete namespace. + * + * @param name the name + */ + public void deleteNamespace(String name) { + client.namespaces().withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + // ========================= + // ---------> POD <--------- + // ========================= + /** + * List pods list. + * + * @param namespaceName the namespace name + * @param labelKey the label key + * @param labelValue the label value + * @return the list + */ + public List listPods(String namespaceName, String labelKey, String labelValue) { + return client.pods().inNamespace(namespaceName).withLabel(labelKey, labelValue).list().getItems(); + } + + /** + * List pods list. + * + * @param namespaceName the namespace name + * @return the list + */ + public List listPods(String namespaceName) { + return client.pods().inNamespace(namespaceName).list().getItems(); + } + + /** + * Returns list of pods by prefix in pod name + * @param namespaceName Namespace name + * @param podNamePrefix prefix with which the name should begin + * @return List of pods + */ + public List listPodsByPrefixInName(String namespaceName, String podNamePrefix) { + return listPods(namespaceName) + .stream().filter(p -> p.getMetadata().getName().startsWith(podNamePrefix)) + .collect(Collectors.toList()); + } + + /** + * Gets pod + * @param namespaceName the namespace name + * @param name the name + * @return the pod + */ + public Pod getPod(String namespaceName, String name) { + return client.pods().inNamespace(namespaceName).withName(name).get(); + } + + /** + * Gets pod. + * + * @param name the name + * @return the pod + */ + public Pod getPod(String name) { + return getPod(getNamespace(), name); + } + + // ================================ + // ---------> DEPLOYMENT <--------- + // ================================ + + /** + * Create or replace deployment deployment. + * + * @param deployment the deployment + * @return the deployment + */ + public Deployment createOrReplaceDeployment(Deployment deployment) { + return client.apps().deployments().inNamespace(deployment.getMetadata().getNamespace()).resource(deployment).create(); + } + + /** + * Gets deployment + * @param namespaceName the namespace name + * @param deploymentName the deployment name + * @return the deployment + */ + public Deployment getDeployment(String namespaceName, String deploymentName) { + return client.apps().deployments().inNamespace(namespaceName).withName(deploymentName).get(); + } + + /** + * Gets deployment status + * @param namespaceName the namespace name + * @param deploymentName the deployment name + * @return the deployment status + */ + public boolean isDeploymentReady(String namespaceName, String deploymentName) { + return client.apps().deployments().inNamespace(namespaceName).withName(deploymentName).isReady(); + } + + /** + * Gets service. + * + * @param namespaceName the namespace name + * @param deploymentName the deployment name + * @return the service + */ + public Service getService(String namespaceName, String deploymentName) { + return client.services().inNamespace(namespaceName).withName(deploymentName).get(); + } + + /** + * Logs in specific namespace string. + * + * @param namespaceName the namespace name + * @param podName the pod name + * @return the string + */ + public String logsInSpecificNamespace(String namespaceName, String podName) { + return client.pods().inNamespace(namespaceName).withName(podName).getLog(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/KubeClusterResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/KubeClusterResource.java new file mode 100644 index 0000000000..d9637652b2 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/KubeClusterResource.java @@ -0,0 +1,139 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.k8s.cluster.KubeCluster; +import io.kroxylicious.systemtests.k8s.cmd.KubeCmdClient; + +/** + * A Junit resource which discovers the running cluster and provides an appropriate KubeClient for it, + * for use with {@code @BeforeAll} (or {@code BeforeEach}. + * For example: + * 

+ *     public static KubeClusterResource testCluster = new KubeClusterResources();
+ *
+ *     @BeforeEach
+ *     void before() {
+ *         testCluster.before();
+ *     }
+ *
+ */ +public class KubeClusterResource { + private static final Logger LOGGER = LoggerFactory.getLogger(KubeClusterResource.class); + private KubeCluster kubeCluster; + private KubeCmdClient cmdClient; + private KubeClient client; + private static KubeClusterResource kubeClusterResource; + private String namespace; + + private KubeClusterResource() { + } + + /** + * Gets instance. + * + * @return the instance + */ + public static synchronized KubeClusterResource getInstance() { + if (kubeClusterResource == null) { + kubeClusterResource = new KubeClusterResource(); + kubeClusterResource.setDefaultNamespace(cmdKubeClient().defaultNamespace()); + LOGGER.info("Cluster default namespace is {}", kubeClusterResource.getNamespace()); + } + return kubeClusterResource; + } + + /** + * Sets default namespace. + * + * @param namespace the namespace + */ + public void setDefaultNamespace(String namespace) { + this.namespace = namespace; + } + + /** + * Gets namespace which is used in Kubernetes clients at the moment + * @return Used namespace + */ + public String getNamespace() { + return namespace; + } + + /** + * Provides appropriate CMD client for running cluster + * @return CMD client + */ + public static KubeCmdClient cmdKubeClient() { + return kubeClusterResource.cmdClient().namespace(kubeClusterResource.getNamespace()); + } + + /** + * Provides appropriate CMD client with expected namespace for running cluster + * @param inNamespace Namespace will be used as a current namespace for client + * @return CMD client with expected namespace in configuration + */ + public static KubeCmdClient cmdKubeClient(String inNamespace) { + return kubeClusterResource.cmdClient().namespace(inNamespace); + } + + /** + * Provides appropriate Kubernetes client for running cluster + * @return Kubernetes client + */ + public static KubeClient kubeClient() { + return kubeClusterResource.client().namespace(kubeClusterResource.getNamespace()); + } + + /** + * Provides appropriate Kubernetes client with expected namespace for running cluster + * @param inNamespace Namespace will be used as a current namespace for client + * @return Kubernetes client with expected namespace in configuration + */ + public static KubeClient kubeClient(String inNamespace) { + return kubeClusterResource.client().namespace(inNamespace); + } + + /** + * Cmd client kube cmd client. + * + * @return the kube cmd client + */ + public synchronized KubeCmdClient cmdClient() { + if (cmdClient == null) { + cmdClient = cluster().defaultCmdClient(); + } + return cmdClient; + } + + /** + * Client kube client. + * + * @return the kube client + */ + public synchronized KubeClient client() { + if (client == null) { + this.client = cluster().defaultClient(); + } + return client; + } + + /** + * Cluster kube cluster. + * + * @return the kube cluster + */ + public synchronized KubeCluster cluster() { + if (kubeCluster == null) { + kubeCluster = KubeCluster.bootstrap(); + } + return kubeCluster; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cluster/KubeCluster.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cluster/KubeCluster.java new file mode 100644 index 0000000000..4627d0d925 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cluster/KubeCluster.java @@ -0,0 +1,78 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.cluster; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; + +import io.kroxylicious.systemtests.k8s.KubeClient; +import io.kroxylicious.systemtests.k8s.cmd.KubeCmdClient; +import io.kroxylicious.systemtests.k8s.exception.NoClusterException; + +/** + * Abstraction for a Kubernetes cluster, for example {@code oc cluster up} or {@code minikube}. + */ +public interface KubeCluster { + + /** + * The constant CONFIG. + */ + Config CONFIG = Config.autoConfigure(null); + + /** Return true iff this kind of cluster installed on the local machine. + * @return the boolean + * */ + boolean isAvailable(); + + /** Return true iff this kind of cluster is running on the local machine + * @return the boolean + * */ + boolean isClusterUp(); + + /** Return a default CMD cmdClient for this kind of cluster. + * @return the kube cmd client + * */ + KubeCmdClient defaultCmdClient(); + + /** + * Default client kube client. + * + * @return the kube client + */ + default KubeClient defaultClient() { + return new KubeClient(new KubernetesClientBuilder().build(), "default"); + } + + /** + * Returns the cluster named by the TEST_CLUSTER environment variable, if set, otherwise finds a cluster that's + * both installed and running. + * @return The cluster. + * @throws NoClusterException If no running cluster was found. + */ + static KubeCluster bootstrap() throws NoClusterException { + Logger logger = LoggerFactory.getLogger(KubeCluster.class); + + KubeCluster cluster = new Kubernetes(); + if (cluster.isAvailable()) { + logger.debug("kubectl is installed"); + if (cluster.isClusterUp()) { + logger.debug("Cluster is running"); + } + else { + throw new NoClusterException("Cluster is not running"); + } + } + else { + throw new NoClusterException("Unable to find a cluster"); + } + + return cluster; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cluster/Kubernetes.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cluster/Kubernetes.java new file mode 100644 index 0000000000..aaff065c27 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cluster/Kubernetes.java @@ -0,0 +1,57 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.cluster; + +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.executor.Exec; +import io.kroxylicious.systemtests.k8s.cmd.KubeCmdClient; +import io.kroxylicious.systemtests.k8s.cmd.Kubectl; +import io.kroxylicious.systemtests.k8s.exception.KubeClusterException; + +/** + * A {@link KubeCluster} implementation for any {@code Kubernetes} cluster. + */ +public class Kubernetes implements KubeCluster { + + /** + * The constant CMD. + */ + public static final String CMD = "kubectl"; + private static final Logger LOGGER = LoggerFactory.getLogger(Kubernetes.class); + + @Override + public boolean isAvailable() { + return Exec.isExecutableOnPath(CMD); + } + + @Override + public boolean isClusterUp() { + List cmd = Arrays.asList(CMD, "cluster-info"); + try { + return Exec.exec(cmd).isSuccess(); + } + catch (KubeClusterException e) { + String commandLine = String.join(" ", cmd); + LOGGER.error("'{}' failed. Please double check connectivity to your cluster!", commandLine, e); + return false; + } + } + + @Override + public KubeCmdClient defaultCmdClient() { + return new Kubectl(); + } + + public String toString() { + return CMD; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/BaseCmdKubeClient.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/BaseCmdKubeClient.java new file mode 100644 index 0000000000..2fe9f8dfb5 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/BaseCmdKubeClient.java @@ -0,0 +1,146 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.cmd; + +import java.io.File; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.executor.Exec; +import io.kroxylicious.systemtests.executor.ExecResult; + +import static java.util.Arrays.asList; + +/** + * The type Base cmd kube client. + * + * @param the type parameter + */ +public abstract class BaseCmdKubeClient> implements KubeCmdClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseCmdKubeClient.class); + + private static final String APPLY = "apply"; + private static final String DELETE = "delete"; + + /** + * The Namespace. + */ + String namespace = defaultNamespace(); + + /** + * The type Context. + */ + protected static class Context implements AutoCloseable { + @Override + public void close() { + // Do nothing + } + } + + private static final Context NOOP = new Context(); + + /** + * Default context context. + * + * @return the context + */ + protected Context defaultContext() { + return NOOP; + } + + /** + * Namespaced command list. + * + * @param rest the rest + * @return the list + */ + protected List namespacedCommand(String... rest) { + return namespacedCommand(asList(rest)); + } + + private List namespacedCommand(List rest) { + List result = new ArrayList<>(); + result.add(cmd()); + result.add("--namespace"); + result.add(namespace); + result.addAll(rest); + return result; + } + + @Override + @SuppressWarnings("unchecked") + public K apply(File... files) { + try (Context context = defaultContext()) { + Map execResults = execRecursive(APPLY, files, Comparator.comparing(File::getName).reversed()); + for (Map.Entry entry : execResults.entrySet()) { + if (!entry.getValue().isSuccess()) { + LOGGER.warn("Failed to apply {}!", entry.getKey().getAbsolutePath()); + LOGGER.debug(entry.getValue().err()); + } + } + return (K) this; + } + } + + @Override + @SuppressWarnings("unchecked") + public K delete(File... files) { + try (Context context = defaultContext()) { + Map execResults = execRecursive(DELETE, files, Comparator.comparing(File::getName).reversed()); + for (Map.Entry entry : execResults.entrySet()) { + if (!entry.getValue().isSuccess()) { + LOGGER.warn("Failed to delete {}!", entry.getKey().getAbsolutePath()); + LOGGER.debug(entry.getValue().err()); + } + } + return (K) this; + } + } + + @Override + @SuppressWarnings("unchecked") + public K deleteByName(String resourceType, String resourceName) { + Exec.exec(namespacedCommand(DELETE, resourceType, resourceName)); + return (K) this; + } + + private Map execRecursive(String subcommand, File[] files, Comparator cmp) { + Map execResults = new HashMap<>(25); + for (File f : files) { + if (f.isFile()) { + if (f.getName().endsWith(".yaml")) { + execResults.put(f, Exec.exec(null, namespacedCommand(subcommand, "-f", f.getAbsolutePath()), 0, false, false, null)); + } + } + else if (f.isDirectory()) { + File[] children = f.listFiles(); + if (children != null) { + Arrays.sort(children, cmp); + execResults.putAll(execRecursive(subcommand, children, cmp)); + } + } + else if (!f.exists()) { + throw new RuntimeException(new NoSuchFileException(f.getPath())); + } + } + return execResults; + } + + @Override + public String toString() { + return cmd(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/KubeCmdClient.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/KubeCmdClient.java new file mode 100644 index 0000000000..5d67f501f7 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/KubeCmdClient.java @@ -0,0 +1,78 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.cmd; + +import java.io.File; + +import static java.util.Arrays.asList; +import static java.util.stream.Collectors.toList; + +/** + * Abstraction for a kubernetes client. + * @param The subtype of KubeClient, for fluency. + */ +public interface KubeCmdClient> { + + /** + * Default namespace string. + * + * @return the string + */ + String defaultNamespace(); + + /** + * Namespace kube cmd client. + * + * @param namespace the namespace + * @return the kube cmd client + */ + KubeCmdClient namespace(String namespace); + + /** Returns namespace for cluster + * @return the string*/ + String namespace(); + + /** Creates the resources in the given files. + * @param files the files + * @return the k + */ + K apply(File... files); + + /** Deletes the resources in the given files. + * @param files the files + * @return the k + */ + K delete(File... files); + + /** + * Delete by name k. + * + * @param resourceType the resource type + * @param resourceName the resource name + private KubeClusterResource() { + } + * @return the k + */ + K deleteByName(String resourceType, String resourceName); + + /** + * Delete k. + * + * @param files the files + * @return the k + */ + default K delete(String... files) { + return delete(asList(files).stream().map(File::new).collect(toList()).toArray(new File[0])); + } + + /** + * Cmd string. + * + * @return the string + */ + String cmd(); +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/Kubectl.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/Kubectl.java new file mode 100644 index 0000000000..e5ca10903f --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/cmd/Kubectl.java @@ -0,0 +1,53 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.cmd; + +/** + * A {@link KubeCmdClient} wrapping {@code kubectl}. + */ +public class Kubectl extends BaseCmdKubeClient { + + /** + * The constant KUBECTL. + */ + public static final String KUBECTL = "kubectl"; + + /** + * Instantiates a new Kubectl. + */ + public Kubectl() { + } + + /** + * Instantiates a new Kubectl. + * + * @param futureNamespace the future namespace + */ + Kubectl(String futureNamespace) { + namespace = futureNamespace; + } + + @Override + public Kubectl namespace(String namespace) { + return new Kubectl(namespace); + } + + @Override + public String namespace() { + return namespace; + } + + @Override + public String defaultNamespace() { + return "default"; + } + + @Override + public String cmd() { + return KUBECTL; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/KubeClusterException.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/KubeClusterException.java new file mode 100644 index 0000000000..1be789d2f6 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/KubeClusterException.java @@ -0,0 +1,88 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.exception; + +import io.kroxylicious.systemtests.executor.ExecResult; + +/** + * The type Kube cluster exception. + */ +public class KubeClusterException extends RuntimeException { + /** + * The Result. + */ + public final ExecResult result; + + /** + * Instantiates a new Kube cluster exception. + * + * @param result the result + * @param s the s + */ + public KubeClusterException(ExecResult result, String s) { + super(s); + this.result = result; + } + + /** + * Instantiates a new Kube cluster exception. + * + * @param cause the cause + */ + public KubeClusterException(Throwable cause) { + super(cause); + this.result = null; + } + + /** + * The type Not found. + */ + public static class NotFound extends KubeClusterException { + + /** + * Instantiates a new Not found. + * + * @param result the result + * @param s the s + */ + public NotFound(ExecResult result, String s) { + super(result, s); + } + } + + /** + * The type Already exists. + */ + public static class AlreadyExists extends KubeClusterException { + + /** + * Instantiates a new Already exists. + * + * @param result the result + * @param s the s + */ + public AlreadyExists(ExecResult result, String s) { + super(result, s); + } + } + + /** + * The type Invalid resource. + */ + public static class InvalidResource extends KubeClusterException { + + /** + * Instantiates a new Invalid resource. + * + * @param result the result + * @param s the s + */ + public InvalidResource(ExecResult result, String s) { + super(result, s); + } + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/NoClusterException.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/NoClusterException.java new file mode 100644 index 0000000000..61ca4e1afb --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/NoClusterException.java @@ -0,0 +1,21 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.exception; + +/** + * The type No cluster exception. + */ +public class NoClusterException extends RuntimeException { + /** + * Instantiates a new No cluster exception. + * + * @param message the message + */ + public NoClusterException(String message) { + super(message); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/WaitException.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/WaitException.java new file mode 100644 index 0000000000..3e1a302744 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/k8s/exception/WaitException.java @@ -0,0 +1,30 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.k8s.exception; + +/** + * The type Wait exception. + */ +public class WaitException extends RuntimeException { + /** + * Instantiates a new Wait exception. + * + * @param message the message + */ + public WaitException(String message) { + super(message); + } + + /** + * Instantiates a new Wait exception. + * + * @param cause the cause + */ + public WaitException(Throwable cause) { + super(cause); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/Resource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/Resource.java new file mode 100644 index 0000000000..8bd568e801 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/Resource.java @@ -0,0 +1,65 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +import io.kroxylicious.systemtests.runner.ThrowableRunner; + +/** + * The type Resource. + * + * @param the type parameter + */ +public class Resource { + /** + * The Throwable runner. + */ + ThrowableRunner throwableRunner; + /** + * The Resource. + */ + T resource; + + /** + * Instantiates a new Resource. + * + * @param throwableRunner the throwable runner + * @param resource the resource + */ + public Resource(ThrowableRunner throwableRunner, T resource) { + this.throwableRunner = throwableRunner; + this.resource = resource; + } + + /** + * Instantiates a new Resource. + * + * @param throwableRunner the throwable runner + */ + public Resource(ThrowableRunner throwableRunner) { + this.throwableRunner = throwableRunner; + } + + /** + * Gets throwable runner. + * + * @return the throwable runner + */ + public ThrowableRunner getThrowableRunner() { + return throwableRunner; + } + + /** + * Gets resource. + * + * @return the resource + */ + public T getResource() { + return resource; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceCondition.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceCondition.java new file mode 100644 index 0000000000..99880cabc8 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceCondition.java @@ -0,0 +1,72 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources; + +import java.util.Objects; +import java.util.function.Predicate; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +/** + * The type Resource condition. + * + * @param the type parameter + */ +public class ResourceCondition { + private final Predicate predicate; + private final String conditionName; + + /** + * Instantiates a new Resource condition. + * + * @param predicate the predicate + * @param conditionName the condition name + */ + public ResourceCondition(Predicate predicate, String conditionName) { + this.predicate = predicate; + this.conditionName = conditionName; + } + + /** + * Gets condition name. + * + * @return the condition name + */ + public String getConditionName() { + return conditionName; + } + + /** + * Gets predicate. + * + * @return the predicate + */ + public Predicate getPredicate() { + return predicate; + } + + /** + * Readiness resource condition. + * + * @param the type parameter + * @param type the type + * @return the resource condition + */ + public static ResourceCondition readiness(ResourceType type) { + return new ResourceCondition<>(type::waitForReadiness, "readiness"); + } + + /** + * Deletion resource condition. + * + * @param the type parameter + * @return the resource condition + */ + public static ResourceCondition deletion() { + return new ResourceCondition<>(Objects::isNull, "deletion"); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceOperation.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceOperation.java new file mode 100644 index 0000000000..2085b94f40 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceOperation.java @@ -0,0 +1,73 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources; + +import java.time.Duration; + +import io.strimzi.api.kafka.model.Kafka; + +import io.kroxylicious.systemtests.Constants; + +/** + * The type Resource operation. + */ +public class ResourceOperation { + /** + * Gets timeout for resource readiness. + * + * @return the timeout for resource readiness + */ + public static long getTimeoutForResourceReadiness() { + return getTimeoutForResourceReadiness("default"); + } + + /** + * Gets timeout for resource readiness. + * + * @param kind the kind + * @return the timeout for resource readiness + */ + public static long getTimeoutForResourceReadiness(String kind) { + return switch (kind) { + case Kafka.RESOURCE_KIND -> Duration.ofMinutes(10).toMillis(); + case Constants.DEPLOYMENT -> Duration.ofMinutes(6).toMillis(); + default -> Duration.ofMinutes(3).toMillis(); + }; + } + + /** + * timeoutForPodsOperation returns a reasonable timeout in milliseconds for a number of Pods in a quorum to roll on update, + * scale up or create + * @param numberOfPods the number of pods + * @return the long + */ + public static long timeoutForPodsOperation(int numberOfPods) { + return Duration.ofMinutes(5).toMillis() * Math.max(1, numberOfPods); + } + + /** + * Gets timeout for resource deletion. + * + * @return the timeout for resource deletion + */ + public static long getTimeoutForResourceDeletion() { + return getTimeoutForResourceDeletion("default"); + } + + /** + * Gets timeout for resource deletion. + * + * @param kind the kind + * @return the timeout for resource deletion + */ + public static long getTimeoutForResourceDeletion(String kind) { + return switch (kind) { + case Kafka.RESOURCE_KIND, Constants.POD_KIND -> Duration.ofMinutes(5).toMillis(); + default -> Duration.ofMinutes(3).toMillis(); + }; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceType.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceType.java new file mode 100644 index 0000000000..fa99c4ac1a --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/ResourceType.java @@ -0,0 +1,57 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +/** + * The interface Resource type. + * + * @param the type parameter + */ +public interface ResourceType { + /** + * Gets kind. + * + * @return the kind + */ + String getKind(); + + /** + * Retrieve resource using Kubernetes API + * @param namespace the namespace + * @param name the name + * @return specific resource with T type. + */ + T get(String namespace, String name); + + /** + * Creates specific resource based on T type using Kubernetes API + * @param resource the resource + */ + void create(T resource); + + /** + * Delete specific resource based on T type using Kubernetes API + * @param resource the resource + */ + void delete(T resource); + + /** + * Update specific resource based on T type using Kubernetes API + * @param resource the resource + */ + void update(T resource); + + /** + * Check if this resource is marked as ready or not with wait. + * + * @param resource the resource + * @return true if ready. + */ + boolean waitForReadiness(T resource); +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousConfigResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousConfigResource.java new file mode 100644 index 0000000000..9cc2168cf7 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousConfigResource.java @@ -0,0 +1,63 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.kroxylicious; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapList; +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.Resource; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceType; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kroxylicious config resource. + */ +public class KroxyliciousConfigResource implements ResourceType { + @Override + public String getKind() { + return Constants.CONFIG_MAP_KIND; + } + + @Override + public ConfigMap get(String namespace, String name) { + return configClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(ConfigMap resource) { + configClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(ConfigMap resource) { + configClient().inNamespace(resource.getMetadata().getNamespace()).withName( + resource.getMetadata().getName()).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public void update(ConfigMap resource) { + configClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(ConfigMap resource) { + return true; + } + + /** + * Config client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> configClient() { + return kubeClient().getClient().resources(ConfigMap.class, ConfigMapList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousDeploymentResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousDeploymentResource.java new file mode 100644 index 0000000000..0097fef1d9 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousDeploymentResource.java @@ -0,0 +1,68 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.kroxylicious; + +import java.util.concurrent.TimeUnit; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DeploymentList; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.Resource; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceOperation; +import io.kroxylicious.systemtests.resources.ResourceType; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kroxylicious deployment resource. + */ +public class KroxyliciousDeploymentResource implements ResourceType { + @Override + public String getKind() { + return Constants.DEPLOYMENT; + } + + @Override + public Deployment get(String namespace, String name) { + return deploymentClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(Deployment resource) { + deploymentClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(Deployment resource) { + deploymentClient().inNamespace(resource.getMetadata().getNamespace()).withName( + resource.getMetadata().getName()).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public void update(Deployment resource) { + deploymentClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(Deployment resource) { + deploymentClient().inNamespace(resource.getMetadata().getNamespace()).withName(resource.getMetadata().getName()) + .waitUntilReady(ResourceOperation.getTimeoutForResourceReadiness(resource.getKind()), TimeUnit.MILLISECONDS); + return true; + } + + /** + * Deployment client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> deploymentClient() { + return kubeClient().getClient().resources(Deployment.class, DeploymentList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousServiceResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousServiceResource.java new file mode 100644 index 0000000000..c81949dd6a --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/kroxylicious/KroxyliciousServiceResource.java @@ -0,0 +1,68 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.kroxylicious; + +import java.util.concurrent.TimeUnit; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceList; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.Resource; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceOperation; +import io.kroxylicious.systemtests.resources.ResourceType; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kroxylicious service resource. + */ +public class KroxyliciousServiceResource implements ResourceType { + @Override + public String getKind() { + return Constants.SERVICE_KIND; + } + + @Override + public Service get(String namespace, String name) { + return serviceClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(Service resource) { + serviceClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(Service resource) { + serviceClient().inNamespace(resource.getMetadata().getNamespace()).withName( + resource.getMetadata().getName()).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public void update(Service resource) { + serviceClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(Service resource) { + serviceClient().inNamespace(resource.getMetadata().getNamespace()).withName(resource.getMetadata().getName()) + .waitUntilReady(ResourceOperation.getTimeoutForResourceReadiness(resource.getKind()), TimeUnit.MILLISECONDS); + return true; + } + + /** + * Kroxylicious service client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> serviceClient() { + return kubeClient().getClient().resources(Service.class, ServiceList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java new file mode 100644 index 0000000000..440cbff1f7 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java @@ -0,0 +1,270 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.manager; + +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.strimzi.api.kafka.model.KafkaTopic; +import io.strimzi.api.kafka.model.Spec; +import io.strimzi.api.kafka.model.status.Status; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.enums.ConditionStatus; +import io.kroxylicious.systemtests.resources.ResourceCondition; +import io.kroxylicious.systemtests.resources.ResourceOperation; +import io.kroxylicious.systemtests.resources.ResourceType; +import io.kroxylicious.systemtests.resources.kroxylicious.KroxyliciousConfigResource; +import io.kroxylicious.systemtests.resources.kroxylicious.KroxyliciousDeploymentResource; +import io.kroxylicious.systemtests.resources.kroxylicious.KroxyliciousServiceResource; +import io.kroxylicious.systemtests.resources.strimzi.KafkaNodePoolResource; +import io.kroxylicious.systemtests.resources.strimzi.KafkaResource; +import io.kroxylicious.systemtests.resources.strimzi.KafkaTopicResource; +import io.kroxylicious.systemtests.resources.strimzi.KafkaUserResource; +import io.kroxylicious.systemtests.utils.TestUtils; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * The type Resource manager. + */ +public class ResourceManager { + private static final Logger LOGGER = LoggerFactory.getLogger(ResourceManager.class); + private static ResourceManager instance; + + private ResourceManager() { + } + + /** + * Gets instance. + * + * @return the instance + */ + public static synchronized ResourceManager getInstance() { + if (instance == null) { + instance = new ResourceManager(); + } + return instance; + } + + private final ResourceType[] resourceTypes = new ResourceType[]{ + new KafkaResource(), + new KafkaTopicResource(), + new KafkaUserResource(), + new KafkaNodePoolResource(), + new KroxyliciousServiceResource(), + new KroxyliciousConfigResource(), + new KroxyliciousDeploymentResource() + }; + + /** + * Create resource without wait. + * + * @param the type parameter + * @param resources the resources + */ + @SafeVarargs + public final void createResourceWithoutWait(T... resources) { + createResource(false, resources); + } + + /** + * Create resource with wait. + * + * @param the type parameter + * @param resources the resources + */ + @SafeVarargs + public final void createResourceWithWait(T... resources) { + createResource(true, resources); + } + + @SafeVarargs + private final void createResource(boolean waitReady, T... resources) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + + LOGGER.info("Creating/Updating {} {}", + resource.getKind(), resource.getMetadata().getName()); + + assert type != null; + type.create(resource); + } + + if (waitReady) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + if (Objects.equals(resource.getKind(), KafkaTopic.RESOURCE_KIND)) { + continue; + } + if (!waitResourceCondition(resource, ResourceCondition.readiness(type))) { + throw new RuntimeException(String.format("Timed out waiting for %s %s/%s to be ready", resource.getKind(), resource.getMetadata().getNamespace(), + resource.getMetadata().getName())); + } + } + } + } + + /** + * Delete resource. + * + * @param the type parameter + * @param resources the resources + */ + @SafeVarargs + public final void deleteResource(T... resources) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + + if (type == null) { + LOGGER.warn("Can't find resource type, please delete it manually"); + continue; + } + + LOGGER.info("Deleting of {} {}/{}", + resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName()); + + try { + type.delete(resource); + if (!waitResourceCondition(resource, ResourceCondition.deletion())) { + throw new RuntimeException( + String.format("Timed out deleting %s %s/%s", resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName())); + } + } + catch (Exception e) { + LOGGER.error("Failed to delete {} {}/{}", resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName(), e); + } + } + } + + /** + * Wait resource condition boolean. + * + * @param the type parameter + * @param resource the resource + * @param condition the condition + * @return the boolean + */ + public final boolean waitResourceCondition(T resource, ResourceCondition condition) { + assertNotNull(resource); + assertNotNull(resource.getMetadata()); + assertNotNull(resource.getMetadata().getName()); + + ResourceType type = findResourceType(resource); + assertNotNull(type); + boolean[] resourceReady = new boolean[1]; + + TestUtils.waitFor( + "resource condition: " + condition.getConditionName() + " to be fulfilled for resource " + resource.getKind() + ":" + resource.getMetadata().getName(), + Constants.GLOBAL_POLL_INTERVAL_MILLIS, ResourceOperation.getTimeoutForResourceReadiness(resource.getKind()), + () -> { + T res = type.get(resource.getMetadata().getNamespace(), resource.getMetadata().getName()); + resourceReady[0] = condition.getPredicate().test(res); + if (!resourceReady[0]) { + type.delete(res); + } + return resourceReady[0]; + }); + + return resourceReady[0]; + } + + @SuppressWarnings(value = "unchecked") + private ResourceType findResourceType(T resource) { + for (ResourceType type : resourceTypes) { + if (type.getKind().equals(resource.getKind())) { + return (ResourceType) type; + } + } + return null; + } + + /** + * Wait until the CR is in desired state + * @param the type parameter + * @param operation - client of CR - for example kafkaClient() + * @param resource - custom resource + * @param resourceTimeout the resource timeout + * @return returns CR + */ + public static > boolean waitForResourceStatusReady(MixedOperation operation, T resource, + long resourceTimeout) { + return waitForResourceStatusReady(operation, resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName(), + ConditionStatus.TRUE, resourceTimeout); + } + + /** + * Wait for resource status boolean. + * + * @param the type parameter + * @param operation the operation + * @param kind the kind + * @param namespace the namespace + * @param name the name + * @param resourceTimeoutMs the resource timeout ms + * @return the boolean + */ + public static > boolean waitForResourceStatusReady(MixedOperation operation, String kind, + String namespace, String name, + long resourceTimeoutMs) { + return waitForResourceStatusReady(operation, kind, namespace, name, ConditionStatus.TRUE, resourceTimeoutMs); + } + + /** + * Wait for resource status boolean. + * + * @param the type parameter + * @param operation the operation + * @param kind the kind + * @param namespace the namespace + * @param name the name + * @param conditionStatus the condition status + * @param resourceTimeoutMs the resource timeout ms + * @return the boolean + */ + public static > boolean waitForResourceStatusReady(MixedOperation operation, String kind, + String namespace, String name, + ConditionStatus conditionStatus, + long resourceTimeoutMs) { + LOGGER.info("Waiting for {}: {}/{} will have desired state 'Ready'", kind, namespace, name); + + TestUtils.waitFor(String.format("%s: %s/%s will have desired state 'Ready'", kind, namespace, name), + Constants.POLL_INTERVAL_FOR_RESOURCE_READINESS_MILLIS, resourceTimeoutMs, + () -> { + final Status status = operation.inNamespace(namespace) + .withName(name) + .get() + .getStatus(); + if (status != null) { + return status.getConditions().stream() + .anyMatch(condition -> condition.getType().equals("Ready") && condition.getStatus().toUpperCase().equals(conditionStatus.toString())); + } + return false; + }); + + LOGGER.info("{}: {}/{} is in desired state 'Ready'", kind, namespace, name); + return true; + } + + /** + * Wait for resource status ready. + * + * @param the type parameter + * @param operation the operation + * @param resource the resource + * @return the boolean + */ + public static > boolean waitForResourceStatusReady(MixedOperation operation, T resource) { + long resourceTimeout = ResourceOperation.getTimeoutForResourceReadiness(resource.getKind()); + return waitForResourceStatusReady(operation, resource, resourceTimeout); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaNodePoolResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaNodePoolResource.java new file mode 100644 index 0000000000..51d7128997 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaNodePoolResource.java @@ -0,0 +1,60 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.strimzi; + +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.strimzi.api.kafka.KafkaNodePoolList; +import io.strimzi.api.kafka.model.nodepool.KafkaNodePool; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceType; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kafka node pool resource. + */ +public class KafkaNodePoolResource implements ResourceType { + @Override + public String getKind() { + return Constants.KAFKA_NODE_POOL_KIND; + } + + @Override + public KafkaNodePool get(String namespace, String name) { + return kafkaNodePoolClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(KafkaNodePool resource) { + kafkaNodePoolClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(KafkaNodePool resource) { + kafkaNodePoolClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).delete(); + } + + @Override + public void update(KafkaNodePool resource) { + kafkaNodePoolClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(KafkaNodePool resource) { + return resource != null; + } + + /** + * Kafka node pool client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> kafkaNodePoolClient() { + return kubeClient().getClient().resources(KafkaNodePool.class, KafkaNodePoolList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaResource.java new file mode 100644 index 0000000000..08befccdc7 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaResource.java @@ -0,0 +1,66 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.strimzi; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.strimzi.api.kafka.KafkaList; +import io.strimzi.api.kafka.model.Kafka; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceOperation; +import io.kroxylicious.systemtests.resources.ResourceType; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kafka resource. + */ +public class KafkaResource implements ResourceType { + + @Override + public String getKind() { + return Constants.KAFKA_KIND; + } + + @Override + public Kafka get(String namespace, String name) { + return kafkaClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(Kafka resource) { + kafkaClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(Kafka resource) { + kafkaClient().inNamespace(resource.getMetadata().getNamespace()).withName( + resource.getMetadata().getName()).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public void update(Kafka resource) { + kafkaClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(Kafka resource) { + return ResourceManager.waitForResourceStatusReady(kafkaClient(), resource, + ResourceOperation.getTimeoutForResourceReadiness(Constants.KAFKA_KIND)); + } + + /** + * Kafka client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> kafkaClient() { + return kubeClient().getClient().resources(Kafka.class, KafkaList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaTopicResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaTopicResource.java new file mode 100644 index 0000000000..a6583aa18d --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaTopicResource.java @@ -0,0 +1,66 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.strimzi; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.strimzi.api.kafka.KafkaTopicList; +import io.strimzi.api.kafka.model.KafkaTopic; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceOperation; +import io.kroxylicious.systemtests.resources.ResourceType; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kafka topic resource. + */ +public class KafkaTopicResource implements ResourceType { + + @Override + public String getKind() { + return Constants.KAFKA_TOPIC_KIND; + } + + @Override + public KafkaTopic get(String namespace, String name) { + return kafkaTopicClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(KafkaTopic resource) { + kafkaTopicClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(KafkaTopic resource) { + kafkaTopicClient().inNamespace(resource.getMetadata().getNamespace()).withName( + resource.getMetadata().getName()).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public void update(KafkaTopic resource) { + kafkaTopicClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(KafkaTopic resource) { + return ResourceManager.waitForResourceStatusReady(kafkaTopicClient(), resource.getKind(), resource.getMetadata().getNamespace(), + resource.getMetadata().getName(), ResourceOperation.getTimeoutForResourceReadiness(resource.getKind())); + } + + /** + * Kafka topic client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> kafkaTopicClient() { + return kubeClient().getClient().resources(KafkaTopic.class, KafkaTopicList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaUserResource.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaUserResource.java new file mode 100644 index 0000000000..9369343044 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/strimzi/KafkaUserResource.java @@ -0,0 +1,64 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.strimzi; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.strimzi.api.kafka.KafkaUserList; +import io.strimzi.api.kafka.model.KafkaUser; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.ResourceType; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Kafka user resource. + */ +public class KafkaUserResource implements ResourceType { + + @Override + public String getKind() { + return Constants.KAFKA_USER_KIND; + } + + @Override + public KafkaUser get(String namespace, String name) { + return kafkaUserClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(KafkaUser resource) { + kafkaUserClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(KafkaUser resource) { + kafkaUserClient().inNamespace(resource.getMetadata().getNamespace()).withName(resource.getMetadata().getName()) + .withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public boolean waitForReadiness(KafkaUser resource) { + return ResourceManager.waitForResourceStatusReady(kafkaUserClient(), resource); + } + + @Override + public void update(KafkaUser kafkaUser) { + kafkaUserClient().inNamespace(kafkaUser.getMetadata().getNamespace()).resource(kafkaUser).update(); + } + + /** + * Kafka user client mixed operation. + * + * @return the mixed operation + */ + public static MixedOperation> kafkaUserClient() { + return kubeClient().getClient().resources(KafkaUser.class, KafkaUserList.class); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/runner/ThrowableRunner.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/runner/ThrowableRunner.java new file mode 100644 index 0000000000..d83d72a758 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/runner/ThrowableRunner.java @@ -0,0 +1,20 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.runner; + +/** + * The interface Throwable runner. + */ +@FunctionalInterface +public interface ThrowableRunner { + /** + * Run. + * + * @throws Exception the exception + */ + void run() throws Exception; +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/steps/KafkaSteps.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/steps/KafkaSteps.java new file mode 100644 index 0000000000..85209d2fb4 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/steps/KafkaSteps.java @@ -0,0 +1,50 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.steps; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; +import io.kroxylicious.systemtests.templates.strimzi.KafkaTopicTemplates; +import io.kroxylicious.systemtests.utils.KafkaUtils; + +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * The type Kafka steps. + */ +public class KafkaSteps { + private static final ResourceManager resourceManager = ResourceManager.getInstance(); + + private KafkaSteps() { + } + + /** + * Create topic. + * + * @param clusterName the cluster name + * @param topicName the topic name + * @param namespace the namespace + * @param partitions the partitions + * @param replicas the replicas + * @param minIsr the min isr + */ + public static void createTopic(String clusterName, String topicName, String namespace, + int partitions, int replicas, int minIsr) { + resourceManager.createResourceWithWait( + KafkaTopicTemplates.defaultTopic(namespace, clusterName, topicName, partitions, replicas, minIsr).build()); + } + + /** + * Restart kafka broker. + * + * @param clusterName the cluster name + */ + public static void restartKafkaBroker(String clusterName) { + clusterName = clusterName + "-kafka"; + assertThat("Broker has not been restarted successfully!", KafkaUtils.restartBroker(Constants.KROXY_DEFAULT_NAMESPACE, clusterName)); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/steps/KroxyliciousSteps.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/steps/KroxyliciousSteps.java new file mode 100644 index 0000000000..0887e815a5 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/steps/KroxyliciousSteps.java @@ -0,0 +1,43 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.steps; + +import io.kroxylicious.systemtests.utils.KafkaUtils; + +/** + * The type Kroxylicious steps. + */ +public class KroxyliciousSteps { + + private KroxyliciousSteps() { + } + + /** + * Produce messages. + * + * @param topicName the topic name + * @param bootstrap the bootstrap + * @param message the message + * @param numberOfMessages the number of messages + */ + public static void produceMessages(String namespace, String topicName, String bootstrap, String message, int numberOfMessages) { + KafkaUtils.produceMessageWithTestClients(namespace, topicName, bootstrap, message, numberOfMessages); + } + + /** + * Consume messages string. + * + * @param topicName the topic name + * @param bootstrap the bootstrap + * @param numberOfMessages the number of messages + * @param timeoutMillis the timeout millis + * @return the string + */ + public static String consumeMessages(String namespace, String topicName, String bootstrap, int numberOfMessages, long timeoutMillis) { + return KafkaUtils.ConsumeMessageWithTestClients(namespace, topicName, bootstrap, numberOfMessages, timeoutMillis); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigTemplates.java new file mode 100644 index 0000000000..4ce13950b5 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigTemplates.java @@ -0,0 +1,88 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.kroxylicious; + +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; + +import io.kroxylicious.systemtests.Constants; + +/** + * The type Kroxylicous config templates. + */ +public class KroxyliciousConfigTemplates { + + /** + * Default kroxylicious config map builder. + * + * @param clusterName the cluster name + * @param namespaceName the namespace name + * @return the config map builder + */ + public static ConfigMapBuilder defaultKroxyConfig(String clusterName, String namespaceName) { + return new ConfigMapBuilder() + .withApiVersion("v1") + .withKind(Constants.CONFIG_MAP_KIND) + .editMetadata() + .withName(Constants.KROXY_CONFIG_NAME) + .withNamespace(namespaceName) + .endMetadata() + .addToData("config.yaml", getDefaultKroxyConfigMap(clusterName)); + } + + /** + * Gets default kroxylicious config map. + * + * @param clusterName the cluster name + * @return the default kroxylicious config map + */ + public static String getDefaultKroxyConfigMap(String clusterName) { + return """ + adminHttp: + endpoints: + prometheus: {} + virtualClusters: + demo: + targetCluster: + bootstrap_servers: %CLUSTER_NAME%-kafka-bootstrap.%NAMESPACE%.svc.cluster.local:9092 + clusterNetworkAddressConfigProvider: + type: PortPerBrokerClusterNetworkAddressConfigProvider + config: + bootstrapAddress: localhost:9292 + brokerAddressPattern: %KROXY_SERVICE_NAME% + logNetwork: false + logFrames: false + """ + .replace("%NAMESPACE%", Constants.KROXY_DEFAULT_NAMESPACE) + .replace("%CLUSTER_NAME%", clusterName) + .replace("%KROXY_SERVICE_NAME%", Constants.KROXY_SERVICE_NAME); + } + + /** + * Gets default external kroxylicious config map. + * + * @param clusterExternalIP the cluster external ip + * @return the default external kroxylicious config map + */ + public static String getDefaultExternalKroxyConfigMap(String clusterExternalIP) { + return """ + adminHttp: + endpoints: + prometheus: {} + virtualClusters: + demo: + targetCluster: + bootstrap_servers: %CLUSTER_EXTERNAL_IP%:9094 + clusterNetworkAddressConfigProvider: + type: PortPerBrokerClusterNetworkAddressConfigProvider + config: + bootstrapAddress: localhost:9292 + logNetwork: false + logFrames: false + """ + .replace("%CLUSTER_EXTERNAL_IP%", clusterExternalIP); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousDeploymentTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousDeploymentTemplates.java new file mode 100644 index 0000000000..b761443cb5 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousDeploymentTemplates.java @@ -0,0 +1,102 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.kroxylicious; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.ContainerPort; +import io.fabric8.kubernetes.api.model.ContainerPortBuilder; +import io.fabric8.kubernetes.api.model.VolumeMount; +import io.fabric8.kubernetes.api.model.VolumeMountBuilder; +import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; + +import io.kroxylicious.systemtests.Constants; + +/** + * The type Kroxylicious deployment templates. + */ +public class KroxyliciousDeploymentTemplates { + + private static final Map kroxyLabelSelector = Map.of("app", "kroxylicious"); + private static final String CONFIG_VOLUME_NAME = "config-volume"; + + /** + * Default kroxylicious deployment deployment builder. + * + * @param namespaceName the namespace name + * @param containerImage the container image + * @param replicas the replicas + * @return the deployment builder + */ + public static DeploymentBuilder defaultKroxyDeployment(String namespaceName, String containerImage, int replicas) { + return new DeploymentBuilder() + .withApiVersion("apps/v1") + .withKind(Constants.DEPLOYMENT) + .withNewMetadata() + .withName(Constants.KROXY_DEPLOYMENT_NAME) + .withNamespace(namespaceName) + .addToLabels(kroxyLabelSelector) + .endMetadata() + .withNewSpec() + .withReplicas(replicas) + .withNewSelector() + .addToMatchLabels(kroxyLabelSelector) + .endSelector() + .withNewTemplate() + .withNewMetadata() + .withLabels(kroxyLabelSelector) + .endMetadata() + .withNewSpec() + .withContainers(new ContainerBuilder() + .withName("kroxylicious") + .withImage(containerImage) + .withImagePullPolicy("Always") + .withArgs("--config", "/opt/kroxylicious/config/config.yaml") + .withPorts(getPlainContainerPortList()) + .withVolumeMounts(getPlainVolumeMountList()) + .build()) + .addNewVolume() + .withName(CONFIG_VOLUME_NAME) + .withNewConfigMap() + .withName(Constants.KROXY_CONFIG_NAME) + .endConfigMap() + .endVolume() + .endSpec() + .endTemplate() + .endSpec(); + } + + private static List getPlainVolumeMountList() { + List volumeMountList = new ArrayList<>(); + volumeMountList.add(new VolumeMountBuilder() + .withName(CONFIG_VOLUME_NAME) + .withMountPath("/opt/kroxylicious/config/config.yaml") + .withSubPath("config.yaml") + .build()); + return volumeMountList; + } + + private static List getPlainContainerPortList() { + List portList = new ArrayList<>(); + portList.add(createContainerPort(9190)); + portList.add(createContainerPort(9292)); + portList.add(createContainerPort(9293)); + portList.add(createContainerPort(9294)); + portList.add(createContainerPort(9295)); + + return portList; + } + + private static ContainerPort createContainerPort(int port) { + return new ContainerPortBuilder() + .withContainerPort(port) + .build(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousServiceTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousServiceTemplates.java new file mode 100644 index 0000000000..da32166ca7 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousServiceTemplates.java @@ -0,0 +1,56 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.kroxylicious; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.ServicePort; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; + +import io.kroxylicious.systemtests.Constants; + +public class KroxyliciousServiceTemplates { + + private static final Map kroxyLabelSelector = Map.of("app", "kroxylicious"); + + public static ServiceBuilder defaultKroxyService(String namespaceName) { + return new ServiceBuilder() + .withApiVersion("v1") + .withKind(Constants.SERVICE_KIND) + .withNewMetadata() + .withName(Constants.KROXY_SERVICE_NAME) + .withNamespace(namespaceName) + .endMetadata() + .editSpec() + .withSelector(kroxyLabelSelector) + .withPorts(getPlainServicePorts()) + .endSpec(); + } + + private static List getPlainServicePorts() { + List servicePorts = new ArrayList<>(); + servicePorts.add(createServicePort("port-9190", 9190, 9190)); + servicePorts.add(createServicePort("port-9292", 9292, 9292)); + servicePorts.add(createServicePort("port-9293", 9293, 9293)); + servicePorts.add(createServicePort("port-9294", 9294, 9294)); + servicePorts.add(createServicePort("port-9295", 9295, 9295)); + return servicePorts; + } + + private static ServicePort createServicePort(String name, int port, int targetPort) { + return new ServicePortBuilder() + .withName(name) + .withProtocol("TCP") + .withPort(port) + .withTargetPort(new IntOrString(targetPort)) + .build(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaNodePoolTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaNodePoolTemplates.java new file mode 100644 index 0000000000..ef5ef9da9e --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaNodePoolTemplates.java @@ -0,0 +1,57 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.strimzi; + +import java.util.Map; + +import io.strimzi.api.kafka.model.nodepool.KafkaNodePoolBuilder; +import io.strimzi.api.kafka.model.nodepool.ProcessRoles; + +import io.kroxylicious.systemtests.Constants; + +/** + * The type Kafka node pool templates. + */ +public class KafkaNodePoolTemplates { + + /** + * Default kafka node pool kafka node pool builder. + * + * @param namespaceName the namespace name + * @param nodePoolName the node pool name + * @param kafkaClusterName the kafka cluster name + * @param kafkaReplicas the kafka replicas + * @return the kafka node pool builder + */ + public static KafkaNodePoolBuilder defaultKafkaNodePool(String namespaceName, String nodePoolName, String kafkaClusterName, int kafkaReplicas) { + return new KafkaNodePoolBuilder() + .withNewMetadata() + .withNamespace(namespaceName) + .withName(nodePoolName) + .withLabels(Map.of(Constants.STRIMZI_CLUSTER_LABEL, kafkaClusterName)) + .endMetadata() + .withNewSpec() + .withReplicas(kafkaReplicas) + .endSpec(); + } + + /** + * Kafka node pool with broker role kafka node pool builder. + * + * @param namespaceName the namespace name + * @param nodePoolName the node pool name + * @param kafkaClusterName the kafka cluster name + * @param kafkaReplicas the kafka replicas + * @return the kafka node pool builder + */ + public static KafkaNodePoolBuilder kafkaNodePoolWithBrokerRole(String namespaceName, String nodePoolName, String kafkaClusterName, int kafkaReplicas) { + return defaultKafkaNodePool(namespaceName, nodePoolName, kafkaClusterName, kafkaReplicas) + .editOrNewSpec() + .addToRoles(ProcessRoles.BROKER) + .endSpec(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaTemplates.java new file mode 100644 index 0000000000..ed3b3fd78c --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaTemplates.java @@ -0,0 +1,132 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.strimzi; + +import io.strimzi.api.kafka.model.KafkaBuilder; +import io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder; +import io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType; +import io.strimzi.api.kafka.model.template.ExternalTrafficPolicy; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.Environment; +import io.kroxylicious.systemtests.enums.LogLevel; +import io.kroxylicious.systemtests.utils.KafkaVersionUtils; + +/** + * The type Kafka templates. + */ +public class KafkaTemplates { + + /** + * Kafka persistent kafka builder. + * + * @param namespaceName the namespace name + * @param clusterName the cluster name + * @param kafkaReplicas the kafka replicas + * @param zkReplicas the zk replicas + * @return the kafka builder + */ + public static KafkaBuilder kafkaPersistent(String namespaceName, String clusterName, int kafkaReplicas, int zkReplicas) { + return defaultKafka(namespaceName, clusterName, kafkaReplicas, zkReplicas) + .editSpec() + .editKafka() + .withNewPersistentClaimStorage() + .withSize("1Gi") + .withDeleteClaim(true) + .endPersistentClaimStorage() + .endKafka() + .editZookeeper() + .withNewPersistentClaimStorage() + .withSize("1Gi") + .withDeleteClaim(true) + .endPersistentClaimStorage() + .endZookeeper() + .endSpec(); + } + + /** + * Kafka persistent with external ip kafka builder. + * + * @param namespaceName the namespace name + * @param clusterName the cluster name + * @param kafkaReplicas the kafka replicas + * @param zkReplicas the zk replicas + * @return the kafka builder + */ + public static KafkaBuilder kafkaPersistentWithExternalIp(String namespaceName, String clusterName, int kafkaReplicas, int zkReplicas) { + return kafkaPersistent(namespaceName, clusterName, kafkaReplicas, zkReplicas) + .editSpec() + .editKafka() + .addToListeners(new GenericKafkaListenerBuilder() + .withName("external") + .withTls(false) + .withPort(9094) + .withType(KafkaListenerType.LOADBALANCER) + .editOrNewConfiguration() + .withExternalTrafficPolicy(ExternalTrafficPolicy.LOCAL) + .endConfiguration() + .build()) + .endKafka() + .endSpec(); + } + + private static KafkaBuilder defaultKafka(String namespaceName, String clusterName, int kafkaReplicas, int zkReplicas) { + return new KafkaBuilder() + .withApiVersion(Constants.KAFKA_API_VERSION_V1BETA2) + .withKind(Constants.KAFKA_KIND) + .withNewMetadata() + .withName(clusterName) + .withNamespace(namespaceName) + .endMetadata() + .editSpec() + .editKafka() + .withVersion(Environment.KAFKA_VERSION) + .withReplicas(kafkaReplicas) + .addToConfig("log.message.format.version", KafkaVersionUtils.getKafkaProtocolVersion(Environment.KAFKA_VERSION)) + .addToConfig("inter.broker.protocol.version", KafkaVersionUtils.getKafkaProtocolVersion(Environment.KAFKA_VERSION)) + .addToConfig("offsets.topic.replication.factor", Math.min(kafkaReplicas, 3)) + .addToConfig("transaction.state.log.min.isr", Math.min(kafkaReplicas, 2)) + .addToConfig("transaction.state.log.replication.factor", Math.min(kafkaReplicas, 3)) + .addToConfig("default.replication.factor", Math.min(kafkaReplicas, 3)) + .addToConfig("min.insync.replicas", Math.min(Math.max(kafkaReplicas - 1, 1), 2)) + .withListeners(new GenericKafkaListenerBuilder() + .withName(Constants.PLAIN_LISTENER_NAME) + .withPort(9092) + .withType(KafkaListenerType.INTERNAL) + .withTls(false) + .build(), + new GenericKafkaListenerBuilder() + .withName(Constants.TLS_LISTENER_NAME) + .withPort(9093) + .withType(KafkaListenerType.INTERNAL) + .withTls(true) + .build()) + .withNewInlineLogging() + .addToLoggers("kafka.root.logger.level", LogLevel.INFO.name()) + .endInlineLogging() + .endKafka() + .editZookeeper() + .withReplicas(zkReplicas) + .withNewInlineLogging() + .addToLoggers("zookeeper.root.logger", LogLevel.INFO.name()) + .endInlineLogging() + .endZookeeper() + .editEntityOperator() + .editUserOperator() + .withNewInlineLogging() + .addToLoggers("rootLogger.level", LogLevel.INFO.name()) + .endInlineLogging() + .endUserOperator() + .editTopicOperator() + .withNewInlineLogging() + .addToLoggers("rootLogger.level", LogLevel.INFO.name()) + .endInlineLogging() + .endTopicOperator() + .endEntityOperator() + .endSpec(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaTopicTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaTopicTemplates.java new file mode 100644 index 0000000000..5adf040aed --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaTopicTemplates.java @@ -0,0 +1,44 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.strimzi; + +import io.strimzi.api.kafka.model.KafkaTopicBuilder; + +import io.kroxylicious.systemtests.Constants; + +/** + * The type Kafka topic templates. + */ +public class KafkaTopicTemplates { + + /** + * Default topic kafka topic builder. + * + * @param topicNamespace the topic namespace + * @param clusterName the cluster name + * @param topicName the topic name + * @param partitions the partitions + * @param replicas the replicas + * @param minIsr the min isr + * @return the kafka topic builder + */ + public static KafkaTopicBuilder defaultTopic(String topicNamespace, String clusterName, String topicName, int partitions, int replicas, int minIsr) { + return new KafkaTopicBuilder() + .withApiVersion(Constants.KAFKA_API_VERSION_V1BETA2) + .withKind(Constants.KAFKA_TOPIC_KIND) + .withNewMetadata() + .withName(topicName) + .withNamespace(topicNamespace) + .addToLabels(Constants.STRIMZI_CLUSTER_LABEL, clusterName) + .endMetadata() + .editSpec() + .withPartitions(partitions) + .withReplicas(replicas) + .addToConfig("min.insync.replicas", minIsr) + .endSpec(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaUserTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaUserTemplates.java new file mode 100644 index 0000000000..5fb84cd4ee --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/strimzi/KafkaUserTemplates.java @@ -0,0 +1,34 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.templates.strimzi; + +import io.strimzi.api.kafka.model.KafkaUserBuilder; + +import io.kroxylicious.systemtests.Constants; + +/** + * The type Kafka user templates. + */ +public class KafkaUserTemplates { + + /** + * Default user kafka user builder. + * + * @param namespaceName the namespace name + * @param clusterName the cluster name + * @param name the name + * @return the kafka user builder + */ + public static KafkaUserBuilder defaultUser(String namespaceName, String clusterName, String name) { + return new KafkaUserBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(namespaceName) + .addToLabels(Constants.STRIMZI_CLUSTER_LABEL, clusterName) + .endMetadata(); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/DeploymentUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/DeploymentUtils.java new file mode 100644 index 0000000000..e17b47cfed --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/DeploymentUtils.java @@ -0,0 +1,136 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.time.Duration; +import java.util.Collections; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; + +import io.kroxylicious.systemtests.Constants; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.cmdKubeClient; +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; +import static org.awaitility.Awaitility.await; + +/** + * The type Deployment utils. + */ +public class DeploymentUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DeploymentUtils.class); + + private static final long READINESS_TIMEOUT = Duration.ofMinutes(6).toMillis(); + private static final long DELETION_TIMEOUT = Duration.ofMinutes(5).toMillis(); + private static final String TEST_LOAD_BALANCER_NAME = "test-load-balancer"; + + /** + * Wait for deployment ready. + * + * @param namespaceName the namespace name + * @param deploymentName the deployment name + */ + public static void waitForDeploymentReady(String namespaceName, String deploymentName) { + LOGGER.info("Waiting for Deployment: {}/{} to be ready", namespaceName, deploymentName); + + TestUtils.waitFor("readiness of Deployment: " + namespaceName + "/" + deploymentName, + Constants.POLL_INTERVAL_FOR_RESOURCE_READINESS_MILLIS, READINESS_TIMEOUT, + () -> kubeClient(namespaceName).isDeploymentReady(namespaceName, deploymentName)); + + LOGGER.info("Deployment: {}/{} is ready", namespaceName, deploymentName); + } + + /** + * Wait until the given Deployment has been deleted. + * @param namespaceName Namespace name + * @param name The name of the Deployment. + */ + public static void waitForDeploymentDeletion(String namespaceName, String name) { + LOGGER.debug("Waiting for Deployment: {}/{} deletion", namespaceName, name); + TestUtils.waitFor("deletion of Deployment: " + namespaceName + "/" + name, Constants.POLL_INTERVAL_FOR_RESOURCE_DELETION_MILLIS, DELETION_TIMEOUT, + () -> { + if (kubeClient(namespaceName).getDeployment(namespaceName, name) == null) { + return true; + } + else { + LOGGER.warn("Deployment: {}/{} is not deleted yet! Triggering force delete by cmd client!", namespaceName, name); + cmdKubeClient(namespaceName).deleteByName(Constants.DEPLOYMENT, name); + return false; + } + }); + LOGGER.debug("Deployment: {}/{} was deleted", namespaceName, name); + } + + /** + * Gets deployment file from url. + * + * @param url the url + * @return the deployment file from url + * @throws IOException the io exception + */ + public static FileInputStream getDeploymentFileFromURL(String url) throws IOException { + File deploymentFile = Files.createTempFile("deploy", ".yaml", TestUtils.getDefaultPosixFilePermissions()).toFile(); + FileUtils.copyURLToFile( + new URL(url), + deploymentFile, + 2000, + 5000); + deploymentFile.deleteOnExit(); + + return new FileInputStream(deploymentFile); + } + + /** + * Check load balancer is working. + * + * @param namespace the namespace + * @return the boolean + */ + public static boolean checkLoadBalancerIsWorking(String namespace) { + Service service = new ServiceBuilder() + .withKind(Constants.SERVICE_KIND) + .withNewMetadata() + .withName(TEST_LOAD_BALANCER_NAME) + .withNamespace(namespace) + .addToLabels("app", "loadbalancer") + .endMetadata() + .withNewSpec() + .addNewPort() + .withPort(8080) + .endPort() + .withSelector(Collections.singletonMap("app", "loadbalancer")) + .withType(Constants.LOAD_BALANCER_TYPE) + .endSpec() + .build(); + kubeClient().getClient().services().inNamespace(namespace).resource(service).create(); + boolean isWorking; + try { + LOGGER.debug("Waiting for the ingress IP to be available..."); + await().atMost(Duration.ofSeconds(10)) + .until(() -> !kubeClient().getService(namespace, TEST_LOAD_BALANCER_NAME).getStatus().getLoadBalancer().getIngress().isEmpty() + && kubeClient().getService(namespace, TEST_LOAD_BALANCER_NAME).getStatus().getLoadBalancer().getIngress().get(0).getIp() != null); + isWorking = true; + } + catch (Exception e) { + isWorking = false; + LOGGER.warn("The ingress IP is not available!"); + LOGGER.warn(e.getMessage()); + } + kubeClient().getClient().apps().deployments().inNamespace(namespace).withName(TEST_LOAD_BALANCER_NAME).delete(); + return isWorking; + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/KafkaUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/KafkaUtils.java new file mode 100644 index 0000000000..152d0227c9 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/KafkaUtils.java @@ -0,0 +1,212 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.utils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.KubernetesClientException; + +import io.kroxylicious.systemtests.Constants; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; +import static org.awaitility.Awaitility.await; + +/** + * The type Kafka utils. + */ +public class KafkaUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaUtils.class); + + /** + * Consume message string. + * + * @param deployNamespace the deploy namespace + * @param topicName the topic name + * @param bootstrap the bootstrap + * @param timeoutMilliseconds the timeout milliseconds + * @return the string + */ + public static String ConsumeMessage(String deployNamespace, String topicName, String bootstrap, int timeoutMilliseconds) { + LOGGER.debug("Consuming messages from '{}' topic", topicName); + + String kafkaConsumerName = "java-kafka-consumer"; + Pod pod = kubeClient().getClient().run().inNamespace(deployNamespace).withNewRunConfig() + .withImage(Constants.STRIMZI_KAFKA_IMAGE) + .withName(kafkaConsumerName) + .withRestartPolicy("Never") + .withCommand("/bin/sh") + .withArgs("-c", + "bin/kafka-console-consumer.sh --bootstrap-server " + bootstrap + " --topic " + topicName + " --from-beginning --timeout-ms " + + timeoutMilliseconds) + .done(); + long deadline = System.currentTimeMillis() + timeoutMilliseconds * 2L; + long timeLeft = deadline; + while (timeLeft > 0) { + try { + Thread.sleep(1000); + timeLeft = deadline - System.currentTimeMillis(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.trace(e.getMessage()); + } + } + String log = kubeClient().logsInSpecificNamespace(deployNamespace, kafkaConsumerName); + kubeClient().getClient().pods().inNamespace(deployNamespace).withName(kafkaConsumerName).delete(); + return log; + } + + /** + * Consume message with test clients. + * + * @param deployNamespace the deploy namespace + * @param topicName the topic name + * @param bootstrap the bootstrap + * @param numOfMessages the num of messages + * @param timeoutMilliseconds the timeout milliseconds + * @return the string + */ + public static String ConsumeMessageWithTestClients(String deployNamespace, String topicName, String bootstrap, int numOfMessages, long timeoutMilliseconds) { + LOGGER.debug("Consuming messages from '{}' topic", topicName); + InputStream file = replaceStringInResourceFile("kafka-consumer-template.yaml", Map.of( + "%BOOTSTRAP_SERVERS%", bootstrap, + "%NAMESPACE%", deployNamespace, + "%TOPIC_NAME%", topicName, + "%MESSAGE_COUNT%", "\"" + numOfMessages + "\"")); + + kubeClient().getClient().load(file).inNamespace(deployNamespace).create(); + String podName = getPodNameByLabel(deployNamespace, "app", Constants.KAFKA_CONSUMER_CLIENT_LABEL, timeoutMilliseconds); + await().ignoreException(KubernetesClientException.class).atMost(Duration.ofMillis(timeoutMilliseconds)).until(() -> { + if (kubeClient().getClient().pods().inNamespace(deployNamespace).withName(podName).get() != null) { + var log = kubeClient().logsInSpecificNamespace(deployNamespace, podName); + return log.contains(" - " + (numOfMessages - 1)); + } + return false; + }); + return kubeClient().logsInSpecificNamespace(deployNamespace, podName); + } + + private static String getPodNameByLabel(String deployNamespace, String labelKey, String labelValue, long timeoutMilliseconds) { + await().atMost(Duration.ofMillis(timeoutMilliseconds)).until(() -> { + var podList = kubeClient().listPods(deployNamespace, labelKey, labelValue); + return !podList.isEmpty(); + }); + var pods = kubeClient().listPods(deployNamespace, labelKey, labelValue); + return pods.get(pods.size() - 1).getMetadata().getName(); + } + + /** + * Produce message. + * + * @param deployNamespace the deploy namespace + * @param topicName the topic name + * @param message the message + * @param bootstrap the bootstrap + */ + public static void ProduceMessage(String deployNamespace, String topicName, String message, String bootstrap) { + kubeClient().getClient().run().inNamespace(deployNamespace).withNewRunConfig() + .withImage(Constants.STRIMZI_KAFKA_IMAGE) + .withName("java-kafka-producer") + .withRestartPolicy("Never") + .withCommand("/bin/sh") + .withArgs("-c", "echo '" + message + "'| bin/kafka-console-producer.sh --bootstrap-server " + bootstrap + " --topic " + topicName) + .done(); + } + + /** + * Produce message with test clients. + * + * @param deployNamespace the deploy namespace + * @param topicName the topic name + * @param bootstrap the bootstrap + * @param message the message + * @param numOfMessages the num of messages + * @return the string + */ + public static String produceMessageWithTestClients(String deployNamespace, String topicName, String bootstrap, String message, int numOfMessages) { + LOGGER.debug("Producing {} messages in '{}' topic", numOfMessages, topicName); + InputStream file = replaceStringInResourceFile("kafka-producer-template.yaml", Map.of( + "%BOOTSTRAP_SERVERS%", bootstrap, + "%NAMESPACE%", deployNamespace, + "%TOPIC_NAME%", topicName, + "%MESSAGE_COUNT%", "\"" + numOfMessages + "\"", + "%MESSAGE%", message)); + kubeClient().getClient().load(file).inNamespace(deployNamespace).create(); + return getPodNameByLabel(deployNamespace, "app", Constants.KAFKA_PRODUCER_CLIENT_LABEL, Duration.ofSeconds(10).toMillis()); + } + + private static InputStream replaceStringInResourceFile(String resourceTemplateFileName, Map replacements) { + Path path = Path.of(Objects.requireNonNull(KafkaUtils.class + .getClassLoader().getResource(resourceTemplateFileName)).getPath()); + Charset charset = StandardCharsets.UTF_8; + + String content; + try { + content = Files.readString(path, charset); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + for (Map.Entry entry : replacements.entrySet()) { + content = content.replaceAll(entry.getKey(), entry.getValue()); + } + return new ByteArrayInputStream(content.getBytes()); + } + + /** + * Restart broker + * + * @param deployNamespace the deploy namespace + * @param clusterName the cluster name + * @return the boolean + */ + public static boolean restartBroker(String deployNamespace, String clusterName) { + String podName = ""; + String podUid = ""; + List kafkaPods = kubeClient().listPods(Constants.KROXY_DEFAULT_NAMESPACE); + for (Pod pod : kafkaPods) { + String tmpName = pod.getMetadata().getName(); + if (tmpName.startsWith(clusterName) && tmpName.endsWith("0")) { + podName = pod.getMetadata().getName(); + podUid = pod.getMetadata().getUid(); + break; + } + } + kubeClient().getClient().pods().inNamespace(deployNamespace).withName(podName).withGracePeriod(0).delete(); + kubeClient().getClient().pods().inNamespace(deployNamespace).withName(podName).waitUntilCondition(Objects::isNull, 60, TimeUnit.SECONDS); + String finalPodName = podName; + await().atMost(Duration.ofMinutes(1)).until(() -> kubeClient().getClient().pods().inNamespace(deployNamespace).withName(finalPodName) != null); + return !Objects.equals(podUid, getPodUid(deployNamespace, podName)); + } + + private static String getPodUid(String deployNamespace, String podName) { + final Pod pod = kubeClient().getPod(deployNamespace, podName); + if (pod != null) { + return pod.getMetadata().getUid(); + } + else { + return ""; + } + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/KafkaVersionUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/KafkaVersionUtils.java new file mode 100644 index 0000000000..7ba3af3e60 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/KafkaVersionUtils.java @@ -0,0 +1,25 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.utils; + +/** + * The type Kafka version utils. + */ +public class KafkaVersionUtils { + + /** + * Gets kafka protocol version. + * + * @param kafkaVersion the kafka version + * @return the kafka protocol version + */ + public static String getKafkaProtocolVersion(String kafkaVersion) { + String[] splitVersion = kafkaVersion.split("\\."); + + return String.format("%s.%s", splitVersion[0], splitVersion[1]); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/NamespaceUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/NamespaceUtils.java new file mode 100644 index 0000000000..6b0afaf3ec --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/NamespaceUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.Constants; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Namespace utils. + */ +public class NamespaceUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(NamespaceUtils.class); + + /** + * Delete namespace with wait. + * + * @param namespace the namespace + */ + public static void deleteNamespaceWithWait(String namespace) { + LOGGER.info("Deleting namespace: {}", namespace); + kubeClient().deleteNamespace(namespace); + TestUtils.waitFor("namespace to be deleted", Constants.GLOBAL_POLL_INTERVAL_MILLIS, Constants.GLOBAL_TIMEOUT_MILLIS, + () -> kubeClient().getNamespace(namespace) == null); + + LOGGER.info("Namespace: {} deleted", namespace); + } + + /** + * Create namespace with wait. + * + * @param namespace the namespace + */ + public static void createNamespaceWithWait(String namespace) { + LOGGER.info("Creating namespace: {}", namespace); + if (kubeClient().getNamespace(namespace) != null) { + LOGGER.warn("Namespace was already created!"); + return; + } + kubeClient().createNamespace(namespace); + TestUtils.waitFor("namespace to be created", Constants.GLOBAL_POLL_INTERVAL_MILLIS, Constants.GLOBAL_TIMEOUT_MILLIS, + () -> kubeClient().getNamespace(namespace) != null); + + LOGGER.info("Namespace: {} created", namespace); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java new file mode 100644 index 0000000000..d20b77692b --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/utils/TestUtils.java @@ -0,0 +1,114 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.utils; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; +import java.util.function.BooleanSupplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.k8s.exception.WaitException; + +/** + * The type Test utils. + */ +public class TestUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); + + /** + * Poll the given {@code ready} function every {@code pollIntervalMs} milliseconds until it returns true, + * or throw a WaitException if it doesn't return true within {@code timeoutMs} milliseconds. + * @param description the description + * @param pollIntervalMs the poll interval ms + * @param timeoutMs the timeout ms + * @param ready the ready + * @return The remaining time left until timeout occurs (helpful if you have several calls which need to share a common timeout), + */ + public static long waitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready) { + return waitFor(description, pollIntervalMs, timeoutMs, ready, () -> { + }); + } + + /** + * Wait for long. + * + * @param description the description + * @param pollIntervalMs the poll interval ms + * @param timeoutMs the timeout ms + * @param ready the ready + * @param onTimeout the on timeout + * @return the long + */ + public static long waitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready, Runnable onTimeout) { + LOGGER.debug("Waiting for {}", description); + long deadline = System.currentTimeMillis() + timeoutMs; + String exceptionMessage = null; + int exceptionCount = 0; + StringWriter stackTraceError = new StringWriter(); + + while (true) { + boolean result; + try { + result = ready.getAsBoolean(); + } + catch (Exception e) { + exceptionMessage = e.getMessage(); + if (++exceptionCount == 1 && exceptionMessage != null) { + // Log the first exception as soon as it occurs + LOGGER.error("Exception waiting for {}, {}", description, exceptionMessage, e); + // log the stacktrace + e.printStackTrace(new PrintWriter(stackTraceError)); + } + result = false; + } + long timeLeft = deadline - System.currentTimeMillis(); + if (result) { + return timeLeft; + } + if (timeLeft <= 0) { + if (exceptionCount > 1) { + LOGGER.error("Exception waiting for {}, {}", description, exceptionMessage); + + if (!stackTraceError.toString().isEmpty()) { + // printing handled stacktrace + LOGGER.error(stackTraceError.toString()); + } + } + onTimeout.run(); + WaitException waitException = new WaitException("Timeout after " + timeoutMs + " ms waiting for " + description); + LOGGER.error(waitException.getMessage()); + throw waitException; + } + long sleepTime = Math.min(pollIntervalMs, timeLeft); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("{} not ready, will try again in {} ms ({}ms till timeout)", description, sleepTime, timeLeft); + } + try { + Thread.sleep(sleepTime); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return deadline - System.currentTimeMillis(); + } + } + } + + /** + * Gets default posix file permissions. + * + * @return the default posix file permissions + */ + public static FileAttribute> getDefaultPosixFilePermissions() { + return PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + } +} diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/AbstractST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/AbstractST.java new file mode 100644 index 0000000000..9dd9c8db69 --- /dev/null +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/AbstractST.java @@ -0,0 +1,122 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests; + +import java.io.IOException; +import java.util.Collections; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.installation.kroxylicious.CertManager; +import io.kroxylicious.systemtests.installation.strimzi.Strimzi; +import io.kroxylicious.systemtests.k8s.KubeClusterResource; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; +import io.kroxylicious.systemtests.utils.NamespaceUtils; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; + +/** + * The type Abstract st. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class AbstractST { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractST.class); + + /** + * The constant cluster. + */ + protected static KubeClusterResource cluster; + /** + * The constant strimziOperator. + */ + protected static Strimzi strimziOperator; + + /** + * The constant certManager. + */ + protected static CertManager certManager; + /** + * The Resource manager. + */ + protected final ResourceManager resourceManager = ResourceManager.getInstance(); + + /** + * Before each test. + * + * @param testInfo the test info + */ + @BeforeEach + void beforeEachTest(TestInfo testInfo) { + LOGGER.info(String.join("", Collections.nCopies(76, "#"))); + LOGGER.info(String.format("%s.%s - STARTED", testInfo.getTestClass().get().getName(), testInfo.getTestMethod().get().getName())); + } + + /** + * Sets up the tests. + * + * @param testInfo the test info + * @throws IOException the io exception + */ + @BeforeAll + static void setup(TestInfo testInfo) throws IOException { + LOGGER.info(String.join("", Collections.nCopies(76, "#"))); + LOGGER.info(String.format("%s Test Suite - STARTED", testInfo.getTestClass().get().getName())); + cluster = KubeClusterResource.getInstance(); + certManager = new CertManager(); + strimziOperator = new Strimzi(Constants.KROXY_DEFAULT_NAMESPACE); + + // simple teardown before all tests + if (kubeClient().getNamespace(Constants.KROXY_DEFAULT_NAMESPACE) != null) { + NamespaceUtils.deleteNamespaceWithWait(Constants.KROXY_DEFAULT_NAMESPACE); + } + if (kubeClient().getNamespace(Constants.CERT_MANAGER_NAMESPACE) != null) { + certManager.delete(); + NamespaceUtils.deleteNamespaceWithWait(Constants.CERT_MANAGER_NAMESPACE); + } + NamespaceUtils.createNamespaceWithWait(Constants.KROXY_DEFAULT_NAMESPACE); + strimziOperator.deploy(); + certManager.deploy(); + } + + /** + * Teardown. + * + * @param testInfo the test info + * @throws IOException the io exception + */ + @AfterAll + static void teardown(TestInfo testInfo) throws IOException { + if (strimziOperator != null) { + strimziOperator.delete(); + } + if (certManager != null) { + certManager.delete(); + } + NamespaceUtils.deleteNamespaceWithWait(Constants.KROXY_DEFAULT_NAMESPACE); + NamespaceUtils.deleteNamespaceWithWait(Constants.CERT_MANAGER_NAMESPACE); + LOGGER.info(String.join("", Collections.nCopies(76, "#"))); + LOGGER.info(String.format("%s Test Suite - FINISHED", testInfo.getTestClass().get().getName())); + } + + /** + * After each test. + * + * @param testInfo the test info + */ + @AfterEach + void afterEachTest(TestInfo testInfo) { + LOGGER.info(String.join("", Collections.nCopies(76, "#"))); + LOGGER.info(String.format("%s.%s - FINISHED", testInfo.getTestClass().get().getName(), testInfo.getTestMethod().get().getName())); + } +} diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/KroxyliciousAppST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/KroxyliciousAppST.java new file mode 100644 index 0000000000..804edf71f1 --- /dev/null +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/KroxyliciousAppST.java @@ -0,0 +1,69 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.installation.kroxylicious.KroxyliciousApp; +import io.kroxylicious.systemtests.templates.strimzi.KafkaTemplates; +import io.kroxylicious.systemtests.utils.DeploymentUtils; + +import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * The Kroxylicious app system tests. + * If using minikube, 'minikube tunnel' shall be executed before these tests + * + * Disabled to focus on kubernetes system tests + */ +@Disabled +public class KroxyliciousAppST extends AbstractST { + private static final Logger LOGGER = LoggerFactory.getLogger(KroxyliciousAppST.class); + private static KroxyliciousApp kroxyliciousApp; + private final String clusterName = "my-external-cluster"; + + /** + * Kroxylicious app is running. + */ + @Test + void kroxyAppIsRunning() { + LOGGER.info("Given local Kroxylicious"); + String clusterIp = kubeClient().getService(Constants.KROXY_DEFAULT_NAMESPACE, clusterName + "-kafka-external-bootstrap").getSpec().getClusterIP(); + kroxyliciousApp = new KroxyliciousApp(clusterIp); + kroxyliciousApp.waitForKroxyliciousProcess(); + assertThat("Kroxylicious app is not running!", kroxyliciousApp.isRunning()); + } + + /** + * Sets before all. + */ + @BeforeAll + void setupBefore() { + assumeTrue(DeploymentUtils.checkLoadBalancerIsWorking(Constants.KROXY_DEFAULT_NAMESPACE), "Load balancer is not working fine, if you are using" + + "minikube please run 'minikube tunnel' before running the tests"); + LOGGER.info("Deploying Kafka in {} namespace", Constants.KROXY_DEFAULT_NAMESPACE); + resourceManager.createResourceWithWait(KafkaTemplates.kafkaPersistentWithExternalIp(Constants.KROXY_DEFAULT_NAMESPACE, clusterName, 3, 3).build()); + } + + /** + * Tear down. + */ + @AfterEach + void tearDown() { + if (kroxyliciousApp != null) { + LOGGER.info("Removing kroxylicious app"); + kroxyliciousApp.stop(); + } + } +} diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/KroxyliciousST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/KroxyliciousST.java new file mode 100644 index 0000000000..12972b0892 --- /dev/null +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/KroxyliciousST.java @@ -0,0 +1,138 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests; + +import java.time.Duration; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.extensions.KroxyliciousExtension; +import io.kroxylicious.systemtests.installation.kroxylicious.Kroxylicious; +import io.kroxylicious.systemtests.steps.KafkaSteps; +import io.kroxylicious.systemtests.steps.KroxyliciousSteps; +import io.kroxylicious.systemtests.templates.strimzi.KafkaTemplates; + +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * The type Acceptance st. + */ +@ExtendWith(KroxyliciousExtension.class) +class KroxyliciousST extends AbstractST { + private static final Logger LOGGER = LoggerFactory.getLogger(KroxyliciousST.class); + private static Kroxylicious kroxylicious; + private final String clusterName = "my-cluster"; + + /** + * Produce and consume message. + * + * @param namespace the namespace + */ + @Test + void produceAndConsumeMessage(String namespace) { + String topicName = "my-topic"; + String message = "Hello-world"; + int numberOfMessages = 1; + String consumedMessage = message + " - " + (numberOfMessages - 1); + + // start Kroxylicious + LOGGER.info("Given Kroxylicious in {} namespace with {} replicas", namespace, 1); + kroxylicious = new Kroxylicious(namespace); + kroxylicious.deployPortPerBrokerPlain(clusterName, 1); + + LOGGER.info("And KafkaTopic in {} namespace", namespace); + KafkaSteps.createTopic(topicName, clusterName, namespace, 1, 1, 1); + + String bootstrap = kroxylicious.getBootstrap(); + + LOGGER.info("When {} messages '{}' are sent to the topic '{}'", numberOfMessages, message, topicName); + KroxyliciousSteps.produceMessages(namespace, topicName, bootstrap, message, numberOfMessages); + + LOGGER.info("Then the {} messages are consumed", numberOfMessages); + String result = KroxyliciousSteps.consumeMessages(namespace, topicName, bootstrap, numberOfMessages, Duration.ofMinutes(2).toMillis()); + LOGGER.info("Received: " + result); + assertThat("'" + consumedMessage + "' message not consumed!", result.contains(consumedMessage)); + } + + /** + * Restart kafka brokers. + * + * @param namespace the namespace + */ + @Test + void restartKafkaBrokers(String namespace) { + String topicName = "my-topic2"; + String message = "Hello-world"; + int numberOfMessages = 20; + String consumedMessage = message + " - " + (numberOfMessages - 1); + + // start Kroxylicious + LOGGER.info("Given Kroxylicious in {} namespace with {} replicas", namespace, 1); + kroxylicious = new Kroxylicious(namespace); + kroxylicious.deployPortPerBrokerPlain(clusterName, 1); + String bootstrap = kroxylicious.getBootstrap(); + + LOGGER.info("And KafkaTopic in {} namespace", namespace); + KafkaSteps.createTopic(topicName, clusterName, namespace, 3, 1, 1); + + LOGGER.info("When {} messages '{}' are sent to the topic '{}'", numberOfMessages, message, topicName); + KroxyliciousSteps.produceMessages(namespace, topicName, bootstrap, message, numberOfMessages); + LOGGER.info("And a kafka broker is restarted"); + KafkaSteps.restartKafkaBroker(clusterName); + + LOGGER.info("Then the {} messages are consumed", numberOfMessages); + String result = KroxyliciousSteps.consumeMessages(namespace, topicName, bootstrap, numberOfMessages, Duration.ofMinutes(10).toMillis()); + LOGGER.info("Received: " + result); + assertThat("'" + consumedMessage + "' message not consumed!", result.contains(consumedMessage)); + } + + /** + * Kroxylicious with replicas. + * + * @param namespace the namespace + */ + @Test + void kroxyWithReplicas(String namespace) { + String topicName = "my-topic3"; + String message = "Hello-world"; + int numberOfMessages = 3; + int replicas = 3; + String consumedMessage = message + " - " + (numberOfMessages - 1); + + // start Kroxylicious + LOGGER.info("Given Kroxylicious in {} namespace with {} replicas", namespace, replicas); + kroxylicious = new Kroxylicious(namespace); + kroxylicious.deployPortPerBrokerPlain(clusterName, replicas); + String bootstrap = kroxylicious.getBootstrap(); + int currentReplicas = kroxylicious.getNumberOfReplicas(); + assertThat("Current replicas: " + currentReplicas + "; expected: " + replicas, currentReplicas == replicas); + + LOGGER.info("And KafkaTopic in {} namespace", namespace); + KafkaSteps.createTopic(topicName, clusterName, namespace, 3, 1, 1); + + LOGGER.info("When {} messages '{}' are sent to the topic '{}'", numberOfMessages, message, topicName); + KroxyliciousSteps.produceMessages(namespace, topicName, bootstrap, message, numberOfMessages); + + LOGGER.info("Then the {} messages are consumed", numberOfMessages); + String result = KroxyliciousSteps.consumeMessages(namespace, topicName, bootstrap, numberOfMessages, Duration.ofMinutes(2).toMillis()); + LOGGER.info("Received: " + result); + assertThat("'" + consumedMessage + "' message not consumed!", result.contains(consumedMessage)); + } + + /** + * Sets before all. + */ + @BeforeAll + void setupBefore() { + LOGGER.info("Deploying Kafka in {} namespace", Constants.KROXY_DEFAULT_NAMESPACE); + resourceManager.createResourceWithWait(KafkaTemplates.kafkaPersistent(Constants.KROXY_DEFAULT_NAMESPACE, clusterName, 3, 3).build()); + } +} diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/extensions/KroxyliciousExtension.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/extensions/KroxyliciousExtension.java new file mode 100644 index 0000000000..610e12a9c2 --- /dev/null +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/extensions/KroxyliciousExtension.java @@ -0,0 +1,74 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.extensions; + +import java.lang.reflect.Parameter; +import java.util.UUID; + +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.utils.NamespaceUtils; + +/** + * The type Kroxylicious extension. + */ +public class KroxyliciousExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { + private static final Logger LOGGER = LoggerFactory.getLogger(KroxyliciousExtension.class); + private static final String K8S_NAMESPACE_KEY = "namespace"; + private static final String EXTENSION_STORE_NAME = "io.kroxylicious.systemtests"; + private final ExtensionContext.Namespace junitNamespace; + + /** + * Instantiates a new Kroxylicious extension. + */ + public KroxyliciousExtension() { + junitNamespace = ExtensionContext.Namespace.create(EXTENSION_STORE_NAME); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return !parameterContext.getParameter().getType().isAssignableFrom(TestInfo.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Parameter parameter = parameterContext.getParameter(); + Class type = parameter.getType(); + LOGGER.trace("test {}: Resolving parameter ({} {})", extensionContext.getUniqueId(), type.getSimpleName(), parameter.getName()); + if (String.class.getTypeName().equals(type.getName())) { + if (parameter.getName().toLowerCase().contains("namespace")) { + return extractK8sNamespace(extensionContext); + } + } + return extensionContext; + } + + @Override + public void afterEach(ExtensionContext extensionContext) { + NamespaceUtils.deleteNamespaceWithWait(extractK8sNamespace(extensionContext)); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) { + final String k8sNamespace = Constants.KROXY_DEFAULT_NAMESPACE + "-" + UUID.randomUUID().toString().replace("-", "").substring(0, 6); + extensionContext.getStore(junitNamespace).put(K8S_NAMESPACE_KEY, k8sNamespace); + NamespaceUtils.createNamespaceWithWait(k8sNamespace); + } + + private String extractK8sNamespace(ExtensionContext extensionContext) { + return extensionContext.getStore(junitNamespace).get(K8S_NAMESPACE_KEY, String.class); + } +} diff --git a/kroxylicious-systemtests/src/test/resources/kafka-consumer-template.yaml b/kroxylicious-systemtests/src/test/resources/kafka-consumer-template.yaml new file mode 100644 index 0000000000..c901e01d21 --- /dev/null +++ b/kroxylicious-systemtests/src/test/resources/kafka-consumer-template.yaml @@ -0,0 +1,44 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: kafka-consumer-client + user-test-app: kafka-clients + name: kafka-consumer-client +spec: + backoffLimit: 0 + completions: 1 + parallelism: 1 + template: + metadata: + labels: + app: kafka-consumer-client + job-name: kafka-consumer-client + user-test-app: kafka-clients + name: kafka-consumer-client + namespace: %NAMESPACE% + spec: + containers: + - env: + - name: BOOTSTRAP_SERVERS + value: %BOOTSTRAP_SERVERS% + - name: TOPIC + value: %TOPIC_NAME% + - name: MESSAGE_COUNT + value: %MESSAGE_COUNT% + - name: GROUP_ID + value: my-group + - name: LOG_LEVEL + value: INFO + - name: CLIENT_TYPE + value: KafkaConsumer + image: quay.io/strimzi-test-clients/test-clients:latest-kafka-3.6.0 + imagePullPolicy: IfNotPresent + name: kafka-consumer-client + restartPolicy: "Never" diff --git a/kroxylicious-systemtests/src/test/resources/kafka-producer-template.yaml b/kroxylicious-systemtests/src/test/resources/kafka-producer-template.yaml new file mode 100644 index 0000000000..3f5856ce22 --- /dev/null +++ b/kroxylicious-systemtests/src/test/resources/kafka-producer-template.yaml @@ -0,0 +1,48 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: kafka-producer-client + user-test-app: kafka-clients + name: kafka-producer-client +spec: + backoffLimit: 0 + completions: 1 + parallelism: 1 + template: + metadata: + labels: + app: kafka-producer-client + job-name: kafka-producer-client + user-test-app: kafka-clients + name: kafka-producer-client + namespace: %NAMESPACE% + spec: + containers: + - env: + - name: BOOTSTRAP_SERVERS + value: %BOOTSTRAP_SERVERS% + - name: DELAY_MS + value: "1000" + - name: TOPIC + value: %TOPIC_NAME% + - name: MESSAGE_COUNT + value: %MESSAGE_COUNT% + - name: MESSAGE + value: %MESSAGE% + - name: PRODUCER_ACKS + value: all + - name: LOG_LEVEL + value: DEBUG + - name: CLIENT_TYPE + value: KafkaProducer + image: quay.io/strimzi-test-clients/test-clients:latest-kafka-3.6.0 + imagePullPolicy: IfNotPresent + name: kafka-producer-client + restartPolicy: "Never" diff --git a/kroxylicious-systemtests/src/test/resources/log4j2-test.properties b/kroxylicious-systemtests/src/test/resources/log4j2-test.properties new file mode 100644 index 0000000000..cb174b73d2 --- /dev/null +++ b/kroxylicious-systemtests/src/test/resources/log4j2-test.properties @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +name = Config + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c:%L - %m%n + +rootLogger.level = DEBUG +rootLogger.appenderRef.console.ref = STDOUT +rootLogger.appenderRef.console.level = INFO +rootLogger.additivity = false diff --git a/pom.xml b/pom.xml index 22aa6ea9ba..3c24401091 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,8 @@ 17 17 + 3.1.0 + 3.10.1 true ${java.version} ${java.test.version} @@ -28,6 +30,7 @@ false ${skipTests} ${skipTests} + ${skipTests} sort @@ -54,9 +57,17 @@ 0.4.14 20.10.1-r0 0.8.11 - + 1.14.9 + + + 0.37.0 + 6.8.1 + 3.4.1 + 3.9.5 + 4.2.0 + 2.15.0 Parent POM @@ -254,6 +265,18 @@ pom import + + org.junit.jupiter + junit-jupiter-engine + test + ${junit.version} + + + org.junit.jupiter + junit-jupiter-params + test + ${junit.version} + org.mockito mockito-bom @@ -262,11 +285,53 @@ import + io.fabric8 + kubernetes-client-api + ${fabric8.kubernetes.version} + + + io.fabric8 + kubernetes-client + runtime + ${fabric8.kubernetes.version} + + + io.fabric8 + kubernetes-httpclient-jdk + runtime + ${fabric8.kubernetes.version} + + + io.strimzi + api + ${strimzi.version} + + + org.apache.maven.enforcer + enforcer-api + ${enforcer.api.version} + + + org.apache.maven + maven-core + ${maven.core.version} + + net.bytebuddy byte-buddy ${byte-buddy.version} test + + org.awaitility + awaitility + ${awaitility.version} + + + commons-io + commons-io + ${commons.io.version} + @@ -283,6 +348,7 @@ kroxylicious-integration-tests kroxylicious-filters kroxylicious-sample + kroxylicious-systemtests @@ -416,6 +482,7 @@ 3.2.2 ${skipITs} + false true @@ -475,6 +542,7 @@ 3.2.2 ${skipUTs} + false random @@ -602,6 +670,27 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-systemtest-isolation + + enforce + + + + + + io.kroxylicious:kroxylicious-systemtests + + + + + + + @@ -749,6 +838,7 @@ kroxylicious https://sonarcloud.io + true check