From 265490fc78539b91ab7ae306ce924f10a4e14d9b Mon Sep 17 00:00:00 2001 From: RJ Duffner Date: Fri, 2 Aug 2024 11:59:17 -0700 Subject: [PATCH] kafka-python Instrument temporary fork, kafka-python-ng inside kafka-python's instrumentation (#2537) * Instrument temporary fork, kafka-python-ng inside kafka-python's instrumentation * Make kafkapython-ng run tests --- .github/workflows/instrumentations_1.yml | 1 + CHANGELOG.md | 4 ++++ instrumentation/README.md | 2 +- .../pyproject.toml | 4 +++- .../instrumentation/kafka/__init__.py | 24 ++++++++++++++++++- .../instrumentation/kafka/package.py | 5 +++- .../test-requirements-ng.txt | 15 ++++++++++++ .../instrumentation/bootstrap_gen.py | 6 ++++- tox.ini | 9 +++++++ 9 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt diff --git a/.github/workflows/instrumentations_1.yml b/.github/workflows/instrumentations_1.yml index 2658f4d4b2..1421d7f9a5 100644 --- a/.github/workflows/instrumentations_1.yml +++ b/.github/workflows/instrumentations_1.yml @@ -40,6 +40,7 @@ jobs: - "util-http" - "fastapislim" - "processor-baggage" + - "kafka-pythonng" os: [ubuntu-20.04] exclude: - python-version: pypy3 diff --git a/CHANGELOG.md b/CHANGELOG.md index f3d9b5e40a..e3b91462ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Added +- `opentelemetry-instrumentation-kafka-python` Instrument temporary fork, kafka-python-ng + inside kafka-python's instrumentation + ([#2537](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2537))) + ## Breaking changes ## Fixed diff --git a/instrumentation/README.md b/instrumentation/README.md index 278182e61f..c45f6ba325 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -24,7 +24,7 @@ | [opentelemetry-instrumentation-grpc](./opentelemetry-instrumentation-grpc) | grpcio ~= 1.27 | No | experimental | [opentelemetry-instrumentation-httpx](./opentelemetry-instrumentation-httpx) | httpx >= 0.18.0 | No | migration | [opentelemetry-instrumentation-jinja2](./opentelemetry-instrumentation-jinja2) | jinja2 >= 2.7, < 4.0 | No | experimental -| [opentelemetry-instrumentation-kafka-python](./opentelemetry-instrumentation-kafka-python) | kafka-python >= 2.0 | No | experimental +| [opentelemetry-instrumentation-kafka-python](./opentelemetry-instrumentation-kafka-python) | kafka-python >= 2.0, < 3.0,kafka-python-ng >= 2.0, < 3.0 | No | experimental | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml b/instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml index f9c0cf8332..c256c71b94 100644 --- a/instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ "opentelemetry-api ~= 1.5", @@ -31,7 +32,8 @@ dependencies = [ [project.optional-dependencies] instruments = [ - "kafka-python >= 2.0", + "kafka-python >= 2.0, < 3.0", + "kafka-python-ng >= 2.0, < 3.0" ] [project.entry-points.opentelemetry_instrumentor] diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py index 8d7378dfdf..b29990d6e3 100644 --- a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/__init__.py @@ -67,6 +67,7 @@ def consume_hook(span, record, args, kwargs): API ___ """ +from importlib.metadata import PackageNotFoundError, distribution from typing import Collection import kafka @@ -74,7 +75,11 @@ def consume_hook(span, record, args, kwargs): from opentelemetry import trace from opentelemetry.instrumentation.instrumentor import BaseInstrumentor -from opentelemetry.instrumentation.kafka.package import _instruments +from opentelemetry.instrumentation.kafka.package import ( + _instruments, + _instruments_kafka_python, + _instruments_kafka_python_ng, +) from opentelemetry.instrumentation.kafka.utils import _wrap_next, _wrap_send from opentelemetry.instrumentation.kafka.version import __version__ from opentelemetry.instrumentation.utils import unwrap @@ -86,6 +91,23 @@ class KafkaInstrumentor(BaseInstrumentor): """ def instrumentation_dependencies(self) -> Collection[str]: + # Determine which package of kafka-python is installed + # Right now there are two packages, kafka-python and kafka-python-ng + # The latter is a fork of the former because the former is connected + # to a pypi namespace that the current maintainers cannot access + # https://github.com/dpkp/kafka-python/issues/2431 + try: + distribution("kafka-python-ng") + return (_instruments_kafka_python_ng,) + except PackageNotFoundError: + pass + + try: + distribution("kafka-python") + return (_instruments_kafka_python,) + except PackageNotFoundError: + pass + return _instruments def _instrument(self, **kwargs): diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py index 04863fb7b4..3a4a5e5de6 100644 --- a/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/src/opentelemetry/instrumentation/kafka/package.py @@ -13,4 +13,7 @@ # limitations under the License. -_instruments = ("kafka-python >= 2.0",) +_instruments_kafka_python = "kafka-python >= 2.0, < 3.0" +_instruments_kafka_python_ng = "kafka-python-ng >= 2.0, < 3.0" + +_instruments = (_instruments_kafka_python, _instruments_kafka_python_ng) diff --git a/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt b/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt new file mode 100644 index 0000000000..7b587031ec --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt @@ -0,0 +1,15 @@ +asgiref==3.7.2 +Deprecated==1.2.14 +importlib-metadata==6.11.0 +iniconfig==2.0.0 +kafka-python-ng==2.2.2 +packaging==24.0 +pluggy==1.5.0 +py-cpuinfo==9.0.0 +pytest==7.4.4 +tomli==2.0.1 +typing_extensions==4.9.0 +wrapt==1.16.0 +zipp==3.19.2 +-e opentelemetry-instrumentation +-e instrumentation/opentelemetry-instrumentation-kafka-python diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py index 0deb99b8bf..1868f6a3a1 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py @@ -105,7 +105,11 @@ "instrumentation": "opentelemetry-instrumentation-jinja2==0.48b0.dev", }, { - "library": "kafka-python >= 2.0", + "library": "kafka-python >= 2.0, < 3.0", + "instrumentation": "opentelemetry-instrumentation-kafka-python==0.48b0.dev", + }, + { + "library": "kafka-python-ng >= 2.0, < 3.0", "instrumentation": "opentelemetry-instrumentation-kafka-python==0.48b0.dev", }, { diff --git a/tox.ini b/tox.ini index 27eaaa9129..2396b4495a 100644 --- a/tox.ini +++ b/tox.ini @@ -350,7 +350,9 @@ envlist = ; opentelemetry-instrumentation-kafka-python py3{8,9,10,11}-test-instrumentation-kafka-python + py3{8,9,10,11,12}-test-instrumentation-kafka-pythonng pypy3-test-instrumentation-kafka-python + pypy3-test-instrumentation-kafka-pythonng lint-instrumentation-kafka-python ; opentelemetry-instrumentation-confluent-kafka @@ -443,6 +445,10 @@ commands_pre = kafka-python: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions kafka-python: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk kafka-python: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements.txt + kafka-pythonng: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api + kafka-pythonng: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions + kafka-pythonng: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk + kafka-pythonng: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt confluent-kafka: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api confluent-kafka: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions @@ -940,6 +946,9 @@ commands = lint-instrumentation-kafka-python: flake8 --config {toxinidir}/.flake8 {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python lint-instrumentation-kafka-python: sh -c "cd instrumentation && pylint --rcfile ../.pylintrc opentelemetry-instrumentation-kafka-python" + ; Test only for kafka-pythonng instrumentation as the only difference between kafka-python and kafka-pythonng is the version of kafka-python + test-instrumentation-kafka-pythonng: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/tests {posargs} + test-instrumentation-confluent-kafka: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-confluent-kafka/tests {posargs} lint-instrumentation-confluent-kafka: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/instrumentation/opentelemetry-instrumentation-confluent-kafka lint-instrumentation-confluent-kafka: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/instrumentation/opentelemetry-instrumentation-confluent-kafka