From 6275b71ab364d347dc4c9b0002ea3bf6f6223d75 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 23 May 2019 13:34:00 -0700 Subject: [PATCH] MINOR: Add 2.0, 2.1 and 2.2 to broker and client compat tests These are important to ensure we don't break compatibility. Author: Ismael Juma Reviewers: Gwen Shapira Closes #6794 from ijuma/update-version-compat-tests --- tests/docker/Dockerfile | 2 ++ .../tests/client/client_compatibility_features_test.py | 5 ++++- .../client/client_compatibility_produce_consume_test.py | 5 ++++- .../tests/core/compatibility_test_new_broker_test.py | 5 ++++- tests/kafkatest/tests/core/upgrade_test.py | 8 +++++++- tests/kafkatest/version.py | 6 +++++- vagrant/base.sh | 5 ++++- 7 files changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index eee48790be2b..41a2336c3bc0 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -54,6 +54,8 @@ RUN mkdir -p "/opt/kafka-1.0.2" && chmod a+rw /opt/kafka-1.0.2 && curl -s "$KAFK RUN mkdir -p "/opt/kafka-1.1.1" && chmod a+rw /opt/kafka-1.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.1.1" RUN mkdir -p "/opt/kafka-2.0.1" && chmod a+rw /opt/kafka-2.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.0.1" RUN mkdir -p "/opt/kafka-2.1.0" && chmod a+rw /opt/kafka-2.1.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.1.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.1.0" +RUN mkdir -p "/opt/kafka-2.1.1" && chmod a+rw /opt/kafka-2.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.1.1" +RUN mkdir -p "/opt/kafka-2.2.0" && chmod a+rw /opt/kafka-2.2.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.2.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.2.0" # Streams test dependencies RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.0.1-test.jar" -o /opt/kafka-0.10.0.1/libs/kafka-streams-0.10.0.1-test.jar diff --git a/tests/kafkatest/tests/client/client_compatibility_features_test.py b/tests/kafkatest/tests/client/client_compatibility_features_test.py index d38657871913..c864e7c493f1 100644 --- a/tests/kafkatest/tests/client/client_compatibility_features_test.py +++ b/tests/kafkatest/tests/client/client_compatibility_features_test.py @@ -23,7 +23,7 @@ from kafkatest.services.zookeeper import ZookeeperService from kafkatest.services.kafka import KafkaService from ducktape.tests.test import Test -from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, V_0_11_0_0, V_0_10_1_0, KafkaVersion +from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, V_0_11_0_0, V_0_10_1_0, KafkaVersion def get_broker_features(broker_version): features = {} @@ -104,6 +104,9 @@ def invoke_compatibility_program(self, features): @parametrize(broker_version=str(LATEST_0_11_0)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_1_1)) + @parametrize(broker_version=str(LATEST_2_0)) + @parametrize(broker_version=str(LATEST_2_1)) + @parametrize(broker_version=str(LATEST_2_2)) def run_compatibility_test(self, broker_version): self.zk.start() self.kafka.set_version(KafkaVersion(broker_version)) diff --git a/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py b/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py index 6e283d277dd7..d06ac52dd232 100644 --- a/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py +++ b/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py @@ -22,7 +22,7 @@ from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int_with_prefix -from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, KafkaVersion +from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, KafkaVersion class ClientCompatibilityProduceConsumeTest(ProduceConsumeValidateTest): """ @@ -59,6 +59,9 @@ def min_cluster_size(self): @parametrize(broker_version=str(LATEST_0_11_0)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_1_1)) + @parametrize(broker_version=str(LATEST_2_0)) + @parametrize(broker_version=str(LATEST_2_1)) + @parametrize(broker_version=str(LATEST_2_2)) def test_produce_consume(self, broker_version): print("running producer_consumer_compat with broker_version = %s" % broker_version) self.kafka.set_version(KafkaVersion(broker_version)) diff --git a/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py b/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py index 8e9b30875184..c7113a78487e 100644 --- a/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py +++ b/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py @@ -23,7 +23,7 @@ from kafkatest.services.zookeeper import ZookeeperService from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int -from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, DEV_BRANCH, KafkaVersion +from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, DEV_BRANCH, KafkaVersion # Compatibility tests for moving to a new broker (e.g., 0.10.x) and using a mix of old and new clients (e.g., 0.9.x) class ClientCompatibilityTestNewBroker(ProduceConsumeValidateTest): @@ -48,6 +48,9 @@ def setUp(self): @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(DEV_BRANCH), compression_types=["none"], timestamp_type=str("LogAppendTime")) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["none"], new_consumer=False, timestamp_type=None) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["snappy"], timestamp_type=str("CreateTime")) + @parametrize(producer_version=str(LATEST_2_2), consumer_version=str(LATEST_2_2), compression_types=["none"], timestamp_type=str("CreateTime")) + @parametrize(producer_version=str(LATEST_2_1), consumer_version=str(LATEST_2_1), compression_types=["zstd"], timestamp_type=str("CreateTime")) + @parametrize(producer_version=str(LATEST_2_0), consumer_version=str(LATEST_2_0), compression_types=["snappy"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_1_1), consumer_version=str(LATEST_1_1), compression_types=["lz4"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_1_0), consumer_version=str(LATEST_1_0), compression_types=["none"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_0_11_0), consumer_version=str(LATEST_0_11_0), compression_types=["gzip"], timestamp_type=str("CreateTime")) diff --git a/tests/kafkatest/tests/core/upgrade_test.py b/tests/kafkatest/tests/core/upgrade_test.py index c9236ee9e8f5..80f18fff2c7b 100644 --- a/tests/kafkatest/tests/core/upgrade_test.py +++ b/tests/kafkatest/tests/core/upgrade_test.py @@ -23,7 +23,7 @@ from kafkatest.services.zookeeper import ZookeeperService from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int -from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, V_0_9_0_0, DEV_BRANCH, KafkaVersion +from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, V_0_9_0_0, DEV_BRANCH, KafkaVersion class TestUpgrade(ProduceConsumeValidateTest): @@ -60,6 +60,12 @@ def perform_upgrade(self, from_kafka_version, to_message_format_version=None): self.kafka.start_node(node) @cluster(num_nodes=6) + @parametrize(from_kafka_version=str(LATEST_2_2), to_message_format_version=None, compression_types=["none"]) + @parametrize(from_kafka_version=str(LATEST_2_2), to_message_format_version=None, compression_types=["zstd"]) + @parametrize(from_kafka_version=str(LATEST_2_1), to_message_format_version=None, compression_types=["none"]) + @parametrize(from_kafka_version=str(LATEST_2_1), to_message_format_version=None, compression_types=["lz4"]) + @parametrize(from_kafka_version=str(LATEST_2_0), to_message_format_version=None, compression_types=["none"]) + @parametrize(from_kafka_version=str(LATEST_2_0), to_message_format_version=None, compression_types=["snappy"]) @parametrize(from_kafka_version=str(LATEST_1_1), to_message_format_version=None, compression_types=["none"]) @parametrize(from_kafka_version=str(LATEST_1_1), to_message_format_version=None, compression_types=["lz4"]) @parametrize(from_kafka_version=str(LATEST_1_0), to_message_format_version=None, compression_types=["none"]) diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index c4a958eb9b48..a3362b32a43f 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -116,4 +116,8 @@ def get_version(node=None): # 2.1.x versions V_2_1_0 = KafkaVersion("2.1.0") -LATEST_2_1 = V_2_1_0 +V_2_1_1 = KafkaVersion("2.1.1") +LATEST_2_1 = V_2_1_1 + +V_2_2_0 = KafkaVersion("2.2.0") +LATEST_2_2 = V_2_2_0 diff --git a/vagrant/base.sh b/vagrant/base.sh index 6ee9660ea18c..6c86815876d7 100755 --- a/vagrant/base.sh +++ b/vagrant/base.sh @@ -131,7 +131,10 @@ get_kafka 2.0.1 2.12 chmod a+rw /opt/kafka-2.0.1 get_kafka 2.1.0 2.12 chmod a+rw /opt/kafka-2.1.0 - +get_kafka 2.1.1 2.12 +chmod a+rw /opt/kafka-2.1.1 +get_kafka 2.2.0 2.12 +chmod a+rw /opt/kafka-2.2.0 # For EC2 nodes, we want to use /mnt, which should have the local disk. On local # VMs, we can just create it if it doesn't exist and use it like we'd use