From 6f004a6bfea2d75d53f6ecb4029288fa50b3454f Mon Sep 17 00:00:00 2001 From: Tommi Vainikainen Date: Fri, 1 Dec 2023 15:10:33 +0200 Subject: [PATCH 1/2] Require kafka codec deps always and test feature support Require kafka codec compression dependencies always instead of optionally, and add unit test to verify the environment. Fixes: #766 --- requirements/requirements-dev.txt | 85 +++++++++++++++------------- requirements/requirements-typing.txt | 16 +++--- requirements/requirements.in | 3 + requirements/requirements.txt | 38 +++++++------ setup.py | 9 ++- tests/unit/test_features.py | 13 +++++ 6 files changed, 96 insertions(+), 68 deletions(-) create mode 100644 tests/unit/test_features.py diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index b810f0704..fd3e2d724 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -6,7 +6,7 @@ # accept-types==0.4.1 # via -r requirements.txt -aiohttp==3.9.0 +aiohttp==3.9.1 # via -r requirements.txt aiokafka==0.8.1 # via -r requirements.txt @@ -14,11 +14,11 @@ aiosignal==1.3.1 # via # -r requirements.txt # aiohttp -anyio==3.7.0 +anyio==4.1.0 # via # -r requirements.txt # watchfiles -async-timeout==4.0.2 +async-timeout==4.0.3 # via # -r requirements.txt # aiohttp @@ -30,40 +30,41 @@ attrs==23.1.0 # hypothesis # jsonschema # referencing + # wmctrl avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py # via -r requirements.txt -blinker==1.6.2 +blinker==1.7.0 # via flask -brotli==1.0.9 +brotli==1.1.0 # via geventhttpclient cachetools==5.3.1 # via -r requirements.txt -certifi==2023.7.22 +certifi==2023.11.17 # via # geventhttpclient # requests # sentry-sdk -charset-normalizer==3.1.0 +charset-normalizer==3.3.2 # via requests -click==8.1.3 +click==8.1.7 # via flask -configargparse==1.5.3 +configargparse==1.7 # via locust confluent-kafka==2.3.0 # via -r requirements.txt -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements.txt # anyio # hypothesis # pytest -execnet==1.9.0 +execnet==2.0.2 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp -filelock==3.12.4 +filelock==3.13.1 # via -r requirements-dev.in -flask==2.3.2 +flask==3.0.0 # via # flask-basicauth # flask-cors @@ -72,7 +73,7 @@ flask-basicauth==0.2.0 # via locust flask-cors==4.0.0 # via locust -frozenlist==1.3.3 +frozenlist==1.4.0 # via # -r requirements.txt # aiohttp @@ -81,13 +82,13 @@ gevent==23.9.1 # via # geventhttpclient # locust -geventhttpclient==2.0.9 +geventhttpclient==2.0.11 # via locust -greenlet==3.0.0rc3 +greenlet==3.0.1 # via gevent hypothesis==6.91.0 # via -r requirements-dev.in -idna==3.4 +idna==3.6 # via # -r requirements.txt # anyio @@ -95,7 +96,7 @@ idna==3.4 # yarl importlib-metadata==6.8.0 # via flask -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via # -r requirements.txt # jsonschema @@ -108,9 +109,9 @@ itsdangerous==2.1.2 # via flask jinja2==3.1.2 # via flask -jsonschema==4.19.2 +jsonschema==4.20.0 # via -r requirements.txt -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.2 # via # -r requirements.txt # jsonschema @@ -118,8 +119,10 @@ kafka-python @ https://github.com/aiven/kafka-python/archive/1b95333c9628152066f # via # -r requirements.txt # aiokafka -locust==2.18.0 +locust==2.19.1 # via -r requirements-dev.in +lz4==4.3.2 + # via -r requirements.txt markdown-it-py==3.0.0 # via # -r requirements.txt @@ -132,7 +135,7 @@ mdurl==0.1.2 # via # -r requirements.txt # markdown-it-py -msgpack==1.0.5 +msgpack==1.0.7 # via locust multidict==6.0.4 # via @@ -141,7 +144,7 @@ multidict==6.0.4 # yarl networkx==3.1 # via -r requirements.txt -packaging==23.1 +packaging==23.2 # via # -r requirements.txt # aiokafka @@ -152,16 +155,16 @@ pkgutil-resolve-name==1.3.10 # via # -r requirements.txt # jsonschema -pluggy==1.2.0 +pluggy==1.3.0 # via pytest protobuf==3.20.3 # via -r requirements.txt -psutil==5.9.5 +psutil==5.9.6 # via # -r requirements-dev.in # locust # pytest-xdist -pygments==2.15.1 +pygments==2.17.2 # via # -r requirements.txt # pdbpp @@ -170,20 +173,22 @@ pyjwt==2.8.0 # via -r requirements.txt pyrepl==0.9.0 # via fancycompleter -pytest==7.4.0 +pytest==7.4.3 # via # -r requirements-dev.in # pytest-timeout # pytest-xdist -pytest-timeout==2.1.0 +pytest-timeout==2.2.0 # via -r requirements-dev.in -pytest-xdist[psutil]==3.3.1 +pytest-xdist[psutil]==3.5.0 # via -r requirements-dev.in python-dateutil==2.8.2 # via -r requirements.txt -pyzmq==25.1.0 +python-snappy==0.6.1 + # via -r requirements.txt +pyzmq==25.1.1 # via locust -referencing==0.30.0 +referencing==0.31.1 # via # -r requirements.txt # jsonschema @@ -196,12 +201,12 @@ rich==13.6.0 # via -r requirements.txt roundrobin==0.0.4 # via locust -rpds-py==0.9.2 +rpds-py==0.13.2 # via # -r requirements.txt # jsonschema # referencing -sentry-sdk==1.31.0 +sentry-sdk==1.38.0 # via -r requirements-dev.in six==1.16.0 # via @@ -219,13 +224,13 @@ tenacity==8.2.3 # via -r requirements.txt tomli==2.0.1 # via pytest -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -r requirements.txt # rich ujson==5.8.0 # via -r requirements.txt -urllib3==2.0.7 +urllib3==2.1.0 # via # requests # sentry-sdk @@ -235,11 +240,11 @@ werkzeug==3.0.1 # via # flask # locust -wmctrl==0.4 +wmctrl==0.5 # via pdbpp -xxhash==3.3.0 +xxhash==3.4.1 # via -r requirements.txt -yarl==1.9.2 +yarl==1.9.3 # via # -r requirements.txt # aiohttp @@ -250,8 +255,10 @@ zipp==3.17.0 # importlib-resources zope-event==5.0 # via gevent -zope-interface==6.0 +zope-interface==6.1 # via gevent +zstandard==0.22.0 + # via -r requirements.txt # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index 0b9a4df67..efec97c7a 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -9,7 +9,7 @@ attrs==23.1.0 # -c requirements-dev.txt # -c requirements.txt # referencing -certifi==2023.7.22 +certifi==2023.11.17 # via # -c requirements-dev.txt # sentry-sdk @@ -17,17 +17,17 @@ mypy==1.7.1 # via -r requirements-typing.in mypy-extensions==1.0.0 # via mypy -referencing==0.30.0 +referencing==0.31.1 # via # -c requirements-dev.txt # -c requirements.txt # types-jsonschema -rpds-py==0.9.2 +rpds-py==0.13.2 # via # -c requirements-dev.txt # -c requirements.txt # referencing -sentry-sdk==1.31.0 +sentry-sdk==1.38.0 # via # -c requirements-dev.txt # -r requirements-typing.in @@ -37,16 +37,16 @@ tomli==2.0.1 # mypy types-cachetools==5.3.0.7 # via -r requirements-typing.in -types-jsonschema==4.19.0.4 +types-jsonschema==4.20.0.0 # via -r requirements-typing.in -types-protobuf==3.20.3 +types-protobuf==3.20.4.6 # via -r requirements-typing.in -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -c requirements-dev.txt # -c requirements.txt # mypy -urllib3==2.0.7 +urllib3==2.1.0 # via # -c requirements-dev.txt # sentry-sdk diff --git a/requirements/requirements.in b/requirements/requirements.in index 751572395..893c9d287 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -5,10 +5,12 @@ aiokafka<1 confluent-kafka==2.3.0 isodate<1 jsonschema<5 +lz4 networkx<4 protobuf<4 pyjwt>=2.4.0<3 python-dateutil<3 +python-snappy tenacity<9 typing-extensions ujson<6 @@ -16,6 +18,7 @@ watchfiles<1 xxhash~=3.3 rich~=13.6.0 cachetools==5.3.1 +zstandard # Patched dependencies # diff --git a/requirements/requirements.txt b/requirements/requirements.txt index ea4ded06c..2b655201d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,15 +6,15 @@ # accept-types==0.4.1 # via -r requirements.in -aiohttp==3.9.0 +aiohttp==3.9.1 # via -r requirements.in aiokafka==0.8.1 # via -r requirements.in aiosignal==1.3.1 # via aiohttp -anyio==3.7.0 +anyio==4.1.0 # via watchfiles -async-timeout==4.0.2 +async-timeout==4.0.3 # via # aiohttp # aiokafka @@ -29,30 +29,32 @@ cachetools==5.3.1 # via -r requirements.in confluent-kafka==2.3.0 # via -r requirements.in -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal -idna==3.4 +idna==3.6 # via # anyio # yarl -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via # jsonschema # jsonschema-specifications isodate==0.6.1 # via -r requirements.in -jsonschema==4.19.2 +jsonschema==4.20.0 # via -r requirements.in -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.2 # via jsonschema kafka-python @ https://github.com/aiven/kafka-python/archive/1b95333c9628152066fb8b1092de9da0433401fd.tar.gz # via # -r requirements.in # aiokafka +lz4==4.3.2 + # via -r requirements.in markdown-it-py==3.0.0 # via rich mdurl==0.1.2 @@ -63,25 +65,27 @@ multidict==6.0.4 # yarl networkx==3.1 # via -r requirements.in -packaging==23.1 +packaging==23.2 # via aiokafka pkgutil-resolve-name==1.3.10 # via jsonschema protobuf==3.20.3 # via -r requirements.in -pygments==2.15.1 +pygments==2.17.2 # via rich pyjwt==2.8.0 # via -r requirements.in python-dateutil==2.8.2 # via -r requirements.in -referencing==0.30.0 +python-snappy==0.6.1 + # via -r requirements.in +referencing==0.31.1 # via # jsonschema # jsonschema-specifications rich==13.6.0 # via -r requirements.in -rpds-py==0.9.2 +rpds-py==0.13.2 # via # jsonschema # referencing @@ -93,7 +97,7 @@ sniffio==1.3.0 # via anyio tenacity==8.2.3 # via -r requirements.in -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -r requirements.in # rich @@ -101,9 +105,11 @@ ujson==5.8.0 # via -r requirements.in watchfiles==0.21.0 # via -r requirements.in -xxhash==3.3.0 +xxhash==3.4.1 # via -r requirements.in -yarl==1.9.2 +yarl==1.9.3 # via aiohttp zipp==3.17.0 # via importlib-resources +zstandard==0.22.0 + # via -r requirements.in diff --git a/setup.py b/setup.py index 9abf3be3c..2e3324ef6 100644 --- a/setup.py +++ b/setup.py @@ -32,15 +32,14 @@ "protobuf", "pyjwt", "python-dateutil", + # compression algorithms supported by AioKafka and KafkaConsumer + "lz4", + "python-snappy", + "zstandard", ], extras_require={ - # compression algorithms supported by KafkaConsumer - "lz4": ["lz4"], "sentry-sdk": ["sentry-sdk>=1.6.0"], - # compression algorithms supported by AioKafka and KafkaConsumer - "snappy": ["python-snappy"], "ujson": ["ujson"], - "zstd": ["python-zstandard"], }, dependency_links=[], package_data={}, diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py new file mode 100644 index 000000000..54e96c2e9 --- /dev/null +++ b/tests/unit/test_features.py @@ -0,0 +1,13 @@ +""" +Copyright (c) 2023 Aiven Ltd +See LICENSE for details +""" +import kafka.codec + + +# Test that the setup has all compression algorithms supported +def test_setup_features() -> None: + assert kafka.codec.has_gzip() + assert kafka.codec.has_lz4() + assert kafka.codec.has_snappy() + assert kafka.codec.has_zstd() From 0d6000e6d1bb444fbfbd2999c043cd73e0e353c2 Mon Sep 17 00:00:00 2001 From: Tommi Vainikainen Date: Fri, 1 Dec 2023 15:42:09 +0200 Subject: [PATCH 2/2] Install snappy-dev headers for codec support via pip --- .github/workflows/lint.yml | 4 ++++ .github/workflows/schema.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 66c1d507e..55fba73f9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,6 +24,8 @@ jobs: with: cache: pip python-version: '3.11' + - name: Install libsnappy-dev + run: sudo apt install libsnappy-dev # required for pylint - run: make karapace/version.py - run: pip install pre-commit @@ -41,6 +43,8 @@ jobs: with: cache: pip python-version: '3.11' + - name: Install libsnappy-dev + run: sudo apt install libsnappy-dev - run: pip install -r requirements/requirements.txt -r requirements/requirements-typing.txt - run: make karapace/version.py - run: mypy diff --git a/.github/workflows/schema.yml b/.github/workflows/schema.yml index 0fd58854b..8b48cdcff 100644 --- a/.github/workflows/schema.yml +++ b/.github/workflows/schema.yml @@ -19,6 +19,8 @@ jobs: cache: pip cache-dependency-path: requirements.txt + - name: Install libsnappy-dev + run: sudo apt install libsnappy-dev - run: pip install -r requirements/requirements.txt # Compare with latest release when running on main. - run: make schema against=$(git describe --abbrev=0 --tags)