From 0b169ae1579c9fbce1689e985fc6955ac51bd3ff Mon Sep 17 00:00:00 2001 From: Gudjon Ragnar Brynjarsson Date: Sun, 3 Mar 2024 17:59:41 +0000 Subject: [PATCH 1/2] use wait_for_logs for kafka container and remove kafka-python dependency --- .../kafka/testcontainers/kafka/__init__.py | 15 ++------- poetry.lock | 32 +++++++------------ pyproject.toml | 3 +- 3 files changed, 15 insertions(+), 35 deletions(-) diff --git a/modules/kafka/testcontainers/kafka/__init__.py b/modules/kafka/testcontainers/kafka/__init__.py index 39983943..57741650 100644 --- a/modules/kafka/testcontainers/kafka/__init__.py +++ b/modules/kafka/testcontainers/kafka/__init__.py @@ -3,11 +3,9 @@ from io import BytesIO from textwrap import dedent -from kafka import KafkaConsumer -from kafka.errors import KafkaError, NoBrokersAvailable, UnrecognizedBrokerVersion from testcontainers.core.container import DockerContainer from testcontainers.core.utils import raise_for_deprecated_parameter -from testcontainers.core.waiting_utils import wait_container_is_ready +from testcontainers.core.waiting_utils import wait_for_logs class KafkaContainer(DockerContainer): @@ -47,13 +45,6 @@ def get_bootstrap_server(self) -> str: port = self.get_exposed_port(self.port) return f"{host}:{port}" - @wait_container_is_ready(UnrecognizedBrokerVersion, NoBrokersAvailable, KafkaError, ValueError) - def _connect(self) -> None: - bootstrap_server = self.get_bootstrap_server() - consumer = KafkaConsumer(group_id="test", bootstrap_servers=[bootstrap_server]) - if not consumer.bootstrap_connected(): - raise KafkaError("Unable to connect with kafka container!") - def tc_start(self) -> None: host = self.get_container_host_ip() port = self.get_exposed_port(self.port) @@ -78,13 +69,13 @@ def tc_start(self) -> None: ) self.create_file(data, KafkaContainer.TC_START_SCRIPT) - def start(self) -> "KafkaContainer": + def start(self, timeout=30) -> "KafkaContainer": script = KafkaContainer.TC_START_SCRIPT command = f'sh -c "while [ ! -f {script} ]; do sleep 0.1; done; sh {script}"' self.with_command(command) super().start() self.tc_start() - self._connect() + wait_for_logs(self, r".*\[KafkaServer id=\d+\] started.*", timeout=timeout) return self def create_file(self, content: bytes, path: str) -> None: diff --git a/poetry.lock b/poetry.lock index 17c05946..06a61d58 100644 --- a/poetry.lock +++ b/poetry.lock @@ -218,8 +218,8 @@ files = [ jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" urllib3 = [ - {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, ] [package.extras] @@ -822,12 +822,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -878,8 +878,8 @@ grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" grpcio = ">=1.51.3,<2.0dev" grpcio-status = ">=1.33.2" proto-plus = [ - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -1285,20 +1285,6 @@ files = [ cryptography = ">=3.4" typing-extensions = ">=4.5.0" -[[package]] -name = "kafka-python" -version = "2.0.2" -description = "Pure Python client for Apache Kafka" -optional = true -python-versions = "*" -files = [ - {file = "kafka-python-2.0.2.tar.gz", hash = "sha256:04dfe7fea2b63726cd6f3e79a2d86e709d608d74406638c5da33a01d45a9d7e3"}, - {file = "kafka_python-2.0.2-py2.py3-none-any.whl", hash = "sha256:2d92418c7cb1c298fa6c7f0fb3519b520d0d7526ac6cb7ae2a4fc65a51a94b6e"}, -] - -[package.extras] -crc32c = ["crc32c"] - [[package]] name = "keyring" version = "24.3.1" @@ -1543,7 +1529,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -2424,6 +2409,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2870,7 +2856,9 @@ python-versions = ">=3.7" files = [ {file = "SQLAlchemy-2.0.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0b148ab0438f72ad21cb004ce3bdaafd28465c4276af66df3b9ecd2037bf252"}, {file = "SQLAlchemy-2.0.28-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bbda76961eb8f27e6ad3c84d1dc56d5bc61ba8f02bd20fcf3450bd421c2fcc9c"}, + {file = "SQLAlchemy-2.0.28-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feea693c452d85ea0015ebe3bb9cd15b6f49acc1a31c28b3c50f4db0f8fb1e71"}, {file = "SQLAlchemy-2.0.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5da98815f82dce0cb31fd1e873a0cb30934971d15b74e0d78cf21f9e1b05953f"}, + {file = "SQLAlchemy-2.0.28-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a5adf383c73f2d49ad15ff363a8748319ff84c371eed59ffd0127355d6ea1da"}, {file = "SQLAlchemy-2.0.28-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56856b871146bfead25fbcaed098269d90b744eea5cb32a952df00d542cdd368"}, {file = "SQLAlchemy-2.0.28-cp310-cp310-win32.whl", hash = "sha256:943aa74a11f5806ab68278284a4ddd282d3fb348a0e96db9b42cb81bf731acdc"}, {file = "SQLAlchemy-2.0.28-cp310-cp310-win_amd64.whl", hash = "sha256:c6c4da4843e0dabde41b8f2e8147438330924114f541949e6318358a56d1875a"}, @@ -2907,7 +2895,9 @@ files = [ {file = "SQLAlchemy-2.0.28-cp38-cp38-win_amd64.whl", hash = "sha256:243feb6882b06a2af68ecf4bec8813d99452a1b62ba2be917ce6283852cf701b"}, {file = "SQLAlchemy-2.0.28-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fc4974d3684f28b61b9a90fcb4c41fb340fd4b6a50c04365704a4da5a9603b05"}, {file = "SQLAlchemy-2.0.28-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87724e7ed2a936fdda2c05dbd99d395c91ea3c96f029a033a4a20e008dd876bf"}, + {file = "SQLAlchemy-2.0.28-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68722e6a550f5de2e3cfe9da6afb9a7dd15ef7032afa5651b0f0c6b3adb8815d"}, {file = "SQLAlchemy-2.0.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:328529f7c7f90adcd65aed06a161851f83f475c2f664a898af574893f55d9e53"}, + {file = "SQLAlchemy-2.0.28-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:df40c16a7e8be7413b885c9bf900d402918cc848be08a59b022478804ea076b8"}, {file = "SQLAlchemy-2.0.28-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:426f2fa71331a64f5132369ede5171c52fd1df1bd9727ce621f38b5b24f48750"}, {file = "SQLAlchemy-2.0.28-cp39-cp39-win32.whl", hash = "sha256:33157920b233bc542ce497a81a2e1452e685a11834c5763933b440fedd1d8e2d"}, {file = "SQLAlchemy-2.0.28-cp39-cp39-win_amd64.whl", hash = "sha256:2f60843068e432311c886c5f03c4664acaef507cf716f6c60d5fde7265be9d7b"}, @@ -3243,7 +3233,7 @@ elasticsearch = [] google = ["google-cloud-pubsub"] influxdb = ["influxdb", "influxdb-client"] k3s = ["kubernetes", "pyyaml"] -kafka = ["kafka-python"] +kafka = [] keycloak = ["python-keycloak"] localstack = ["boto3"] minio = ["minio"] @@ -3262,4 +3252,4 @@ selenium = ["selenium"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "55534a498e0ae69beb3eba43e055910e656f67de80ec72400c4a535f91b2be56" +content-hash = "118e56b91645ea340d5ddc00b543b71ca177ec28d52bb8aeb98491b7616ba09a" diff --git a/pyproject.toml b/pyproject.toml index b9ac9d7c..2906dafb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,6 @@ influxdb = { version = "*", optional = true } influxdb-client = { version = "*", optional = true } kubernetes = { version = "*", optional = true } pyyaml = { version = "*", optional = true } -kafka-python = { version = "*", optional = true } python-keycloak = { version = "*", optional = true } boto3 = { version = "*", optional = true } minio = { version = "*", optional = true } @@ -93,7 +92,7 @@ elasticsearch = [] google = ["google-cloud-pubsub"] influxdb = ["influxdb", "influxdb-client"] k3s = ["kubernetes", "pyyaml"] -kafka = ["kafka-python"] +kafka = [] keycloak = ["python-keycloak"] localstack = ["boto3"] minio = ["minio"] From 728a22c20fc7f792a872595d97c8411a334d8a00 Mon Sep 17 00:00:00 2001 From: Gudjon Ragnar Brynjarsson Date: Mon, 4 Mar 2024 09:59:56 +0000 Subject: [PATCH 2/2] Add kafka python as dev dependency --- poetry.lock | 16 +++++++++++++++- pyproject.toml | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 06a61d58..d27c491a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1285,6 +1285,20 @@ files = [ cryptography = ">=3.4" typing-extensions = ">=4.5.0" +[[package]] +name = "kafka-python" +version = "2.0.2" +description = "Pure Python client for Apache Kafka" +optional = false +python-versions = "*" +files = [ + {file = "kafka-python-2.0.2.tar.gz", hash = "sha256:04dfe7fea2b63726cd6f3e79a2d86e709d608d74406638c5da33a01d45a9d7e3"}, + {file = "kafka_python-2.0.2-py2.py3-none-any.whl", hash = "sha256:2d92418c7cb1c298fa6c7f0fb3519b520d0d7526ac6cb7ae2a4fc65a51a94b6e"}, +] + +[package.extras] +crc32c = ["crc32c"] + [[package]] name = "keyring" version = "24.3.1" @@ -3252,4 +3266,4 @@ selenium = ["selenium"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "118e56b91645ea340d5ddc00b543b71ca177ec28d52bb8aeb98491b7616ba09a" +content-hash = "c092494be845c5f76ff36892af1fd89c23e9445fd4a1014da8f3335c9862f240" diff --git a/pyproject.toml b/pyproject.toml index 2906dafb..abf55d92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,7 @@ anyio = "^4.3.0" psycopg2-binary = "*" pg8000 = "*" sqlalchemy = "*" - +kafka-python = "^2.0.2" [[tool.poetry.source]] name = "PyPI"