Require kafka codec deps always and test feature support

.github/workflows/lint.yml

@@ -24,6 +24,8 @@ jobs:
       with:
         cache: pip
         python-version: '3.11'
+    - name: Install libsnappy-dev
+      run: sudo apt install libsnappy-dev
     # required for pylint
     - run: make karapace/version.py
     - run: pip install pre-commit
@@ -41,6 +43,8 @@ jobs:
       with:
         cache: pip
         python-version: '3.11'
+    - name: Install libsnappy-dev
+      run: sudo apt install libsnappy-dev
     - run: pip install -r requirements/requirements.txt -r requirements/requirements-typing.txt
     - run: make karapace/version.py
     - run: mypy

.github/workflows/schema.yml

@@ -19,6 +19,8 @@ jobs:
           cache: pip
           cache-dependency-path:
             requirements.txt
+      - name: Install libsnappy-dev
+        run: sudo apt install libsnappy-dev
       - run: pip install -r requirements/requirements.txt
       # Compare with latest release when running on main.
       - run: make schema against=$(git describe --abbrev=0 --tags)

requirements/requirements-dev.txt

@@ -6,19 +6,19 @@
 #
 accept-types==0.4.1
     # via -r requirements.txt
-aiohttp==3.9.0
+aiohttp==3.9.1
     # via -r requirements.txt
 aiokafka==0.8.1
     # via -r requirements.txt
 aiosignal==1.3.1
     # via
     #   -r requirements.txt
     #   aiohttp
-anyio==3.7.0
+anyio==4.1.0
     # via
     #   -r requirements.txt
     #   watchfiles
-async-timeout==4.0.2
+async-timeout==4.0.3
     # via
     #   -r requirements.txt
     #   aiohttp
@@ -30,40 +30,41 @@ attrs==23.1.0
     #   hypothesis
     #   jsonschema
     #   referencing
+    #   wmctrl
 avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py
     # via -r requirements.txt
-blinker==1.6.2
+blinker==1.7.0
     # via flask
-brotli==1.0.9
+brotli==1.1.0
     # via geventhttpclient
 cachetools==5.3.1
     # via -r requirements.txt
-certifi==2023.7.22
+certifi==2023.11.17
     # via
     #   geventhttpclient
     #   requests
     #   sentry-sdk
-charset-normalizer==3.1.0
+charset-normalizer==3.3.2
     # via requests
-click==8.1.3
+click==8.1.7
     # via flask
-configargparse==1.5.3
+configargparse==1.7
     # via locust
 confluent-kafka==2.3.0
     # via -r requirements.txt
-exceptiongroup==1.1.3
+exceptiongroup==1.2.0
     # via
     #   -r requirements.txt
     #   anyio
     #   hypothesis
     #   pytest
-execnet==1.9.0
+execnet==2.0.2
     # via pytest-xdist
 fancycompleter==0.9.1
     # via pdbpp
-filelock==3.12.4
+filelock==3.13.1
     # via -r requirements-dev.in
-flask==2.3.2
+flask==3.0.0
     # via
     #   flask-basicauth
     #   flask-cors
@@ -72,7 +73,7 @@ flask-basicauth==0.2.0
     # via locust
 flask-cors==4.0.0
     # via locust
-frozenlist==1.3.3
+frozenlist==1.4.0
     # via
     #   -r requirements.txt
     #   aiohttp
@@ -81,21 +82,21 @@ gevent==23.9.1
     # via
     #   geventhttpclient
     #   locust
-geventhttpclient==2.0.9
+geventhttpclient==2.0.11
     # via locust
-greenlet==3.0.0rc3
+greenlet==3.0.1
     # via gevent
 hypothesis==6.91.0
     # via -r requirements-dev.in
-idna==3.4
+idna==3.6
     # via
     #   -r requirements.txt
     #   anyio
     #   requests
     #   yarl
 importlib-metadata==6.8.0
     # via flask
-importlib-resources==6.1.0
+importlib-resources==6.1.1
     # via
     #   -r requirements.txt
     #   jsonschema
@@ -108,18 +109,20 @@ itsdangerous==2.1.2
     # via flask
 jinja2==3.1.2
     # via flask
-jsonschema==4.19.2
+jsonschema==4.20.0
     # via -r requirements.txt
-jsonschema-specifications==2023.7.1
+jsonschema-specifications==2023.11.2
     # via
     #   -r requirements.txt
     #   jsonschema
 kafka-python @ https://github.com/aiven/kafka-python/archive/1b95333c9628152066fb8b1092de9da0433401fd.tar.gz
     # via
     #   -r requirements.txt
     #   aiokafka
-locust==2.18.0
+locust==2.19.1
     # via -r requirements-dev.in
+lz4==4.3.2
+    # via -r requirements.txt
 markdown-it-py==3.0.0
     # via
     #   -r requirements.txt
@@ -132,7 +135,7 @@ mdurl==0.1.2
     # via
     #   -r requirements.txt
     #   markdown-it-py
-msgpack==1.0.5
+msgpack==1.0.7
     # via locust
 multidict==6.0.4
     # via
@@ -141,7 +144,7 @@ multidict==6.0.4
     #   yarl
 networkx==3.1
     # via -r requirements.txt
-packaging==23.1
+packaging==23.2
     # via
     #   -r requirements.txt
     #   aiokafka
@@ -152,16 +155,16 @@ pkgutil-resolve-name==1.3.10
     # via
     #   -r requirements.txt
     #   jsonschema
-pluggy==1.2.0
+pluggy==1.3.0
     # via pytest
 protobuf==3.20.3
     # via -r requirements.txt
-psutil==5.9.5
+psutil==5.9.6
     # via
     #   -r requirements-dev.in
     #   locust
     #   pytest-xdist
-pygments==2.15.1
+pygments==2.17.2
     # via
     #   -r requirements.txt
     #   pdbpp
@@ -170,20 +173,22 @@ pyjwt==2.8.0
     # via -r requirements.txt
 pyrepl==0.9.0
     # via fancycompleter
-pytest==7.4.0
+pytest==7.4.3
     # via
     #   -r requirements-dev.in
     #   pytest-timeout
     #   pytest-xdist
-pytest-timeout==2.1.0
+pytest-timeout==2.2.0
     # via -r requirements-dev.in
-pytest-xdist[psutil]==3.3.1
+pytest-xdist[psutil]==3.5.0
     # via -r requirements-dev.in
 python-dateutil==2.8.2
     # via -r requirements.txt
-pyzmq==25.1.0
+python-snappy==0.6.1
+    # via -r requirements.txt
+pyzmq==25.1.1
     # via locust
-referencing==0.30.0
+referencing==0.31.1
     # via
     #   -r requirements.txt
     #   jsonschema
@@ -196,12 +201,12 @@ rich==13.6.0
     # via -r requirements.txt
 roundrobin==0.0.4
     # via locust
-rpds-py==0.9.2
+rpds-py==0.13.2
     # via
     #   -r requirements.txt
     #   jsonschema
     #   referencing
-sentry-sdk==1.31.0
+sentry-sdk==1.38.0
     # via -r requirements-dev.in
 six==1.16.0
     # via
@@ -219,13 +224,13 @@ tenacity==8.2.3
     # via -r requirements.txt
 tomli==2.0.1
     # via pytest
-typing-extensions==4.6.3
+typing-extensions==4.8.0
     # via
     #   -r requirements.txt
     #   rich
 ujson==5.8.0
     # via -r requirements.txt
-urllib3==2.0.7
+urllib3==2.1.0
     # via
     #   requests
     #   sentry-sdk
@@ -235,11 +240,11 @@ werkzeug==3.0.1
     # via
     #   flask
     #   locust
-wmctrl==0.4
+wmctrl==0.5
     # via pdbpp
-xxhash==3.3.0
+xxhash==3.4.1
     # via -r requirements.txt
-yarl==1.9.2
+yarl==1.9.3
     # via
     #   -r requirements.txt
     #   aiohttp
@@ -250,8 +255,10 @@ zipp==3.17.0
     #   importlib-resources
 zope-event==5.0
     # via gevent
-zope-interface==6.0
+zope-interface==6.1
     # via gevent
+zstandard==0.22.0
+    # via -r requirements.txt
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/requirements-typing.txt

@@ -9,25 +9,25 @@ attrs==23.1.0
     #   -c requirements-dev.txt
     #   -c requirements.txt
     #   referencing
-certifi==2023.7.22
+certifi==2023.11.17
     # via
     #   -c requirements-dev.txt
     #   sentry-sdk
 mypy==1.7.1
     # via -r requirements-typing.in
 mypy-extensions==1.0.0
     # via mypy
-referencing==0.30.0
+referencing==0.31.1
     # via
     #   -c requirements-dev.txt
     #   -c requirements.txt
     #   types-jsonschema
-rpds-py==0.9.2
+rpds-py==0.13.2
     # via
     #   -c requirements-dev.txt
     #   -c requirements.txt
     #   referencing
-sentry-sdk==1.31.0
+sentry-sdk==1.38.0
     # via
     #   -c requirements-dev.txt
     #   -r requirements-typing.in
@@ -37,16 +37,16 @@ tomli==2.0.1
     #   mypy
 types-cachetools==5.3.0.7
     # via -r requirements-typing.in
-types-jsonschema==4.19.0.4
+types-jsonschema==4.20.0.0
     # via -r requirements-typing.in
-types-protobuf==3.20.3
+types-protobuf==3.20.4.6
     # via -r requirements-typing.in
-typing-extensions==4.6.3
+typing-extensions==4.8.0
     # via
     #   -c requirements-dev.txt
     #   -c requirements.txt
     #   mypy
-urllib3==2.0.7
+urllib3==2.1.0
     # via
     #   -c requirements-dev.txt
     #   sentry-sdk

requirements/requirements.in

@@ -5,17 +5,20 @@ aiokafka<1
 confluent-kafka==2.3.0
 isodate<1
 jsonschema<5
+lz4
 networkx<4
 protobuf<4
 pyjwt>=2.4.0<3
 python-dateutil<3
+python-snappy
 tenacity<9
 typing-extensions
 ujson<6
 watchfiles<1
 xxhash~=3.3
 rich~=13.6.0
 cachetools==5.3.1
+zstandard
 
 # Patched dependencies
 #