[Chore] Datadog CI integration (#403)

## Problem To continue driving performance improvements in the SDK and avoid major regressions, we need to track performance over time. First step in that is figuring out how to send data over to datadog with ddtrace so we can begin to have a historical overview of how perf is changing over time. Thanks to @ssmith-pc for pointing me in the right direction on how to get started with this stuff. ## Solution - Add ddtrace dev dependency - Adjust CI configurations to set env variables expected by ddtrace Probably later I will need to: - Develop some benchmark tests specifically exercising the areas where performance is most critical. - Setup some dashboards on the datadog side where we can easily find and interpret the data Not sure if this link will be visible to others, but here's where I see data flowing into datadog https://app.datadoghq.com/ci/test-runs?saved-view-id=3068550 ## Type of Change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update - [x] Infrastructure change (CI configs, etc) - [ ] Non-code change (docs, etc) - [ ] None of the above: (explain here)
pinecone-io · Oct 25, 2024 · efc751d · efc751d
1 parent d1fda39
commit efc751d
Show file tree

Hide file tree

Showing 8 changed files with 332 additions and 23 deletions.
diff --git a/.github/actions/test-data-plane/action.yaml b/.github/actions/test-data-plane/action.yaml
@@ -29,6 +29,9 @@ inputs:
     description: 'The version of Python to use'
     required: false
     default: '3.9'
+  DATADOG_API_KEY:
+    description: 'The Datadog API key'
+    required: true
 
 outputs:
   index_name:
@@ -52,10 +55,15 @@ runs:
     - name: Run data plane tests
       id: data-plane-tests
       shell: bash
-      run: poetry run pytest tests/integration/data
+      run: poetry run pytest tests/integration/data --ddtrace
       env:
+        DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+        DD_API_KEY: ${{ inputs.DATADOG_API_KEY }}
+        DD_SITE: datadoghq.com
+        DD_ENV: ci
+        DD_SERVICE: pinecone-python-client
         PINECONE_API_KEY: ${{ inputs.PINECONE_API_KEY }}
         USE_GRPC: ${{ inputs.use_grpc }}
         METRIC: ${{ inputs.metric }}
         SPEC: ${{ inputs.spec }}
-        FRESHNESS_TIMEOUT_SECONDS: ${{ inputs.freshness_timeout_seconds }}
+        FRESHNESS_TIMEOUT_SECONDS: ${{ inputs.freshness_timeout_seconds }}
diff --git a/.github/actions/test-dependency-grpc/action.yaml b/.github/actions/test-dependency-grpc/action.yaml
@@ -42,25 +42,25 @@ runs:
     - name: Install grpcio ${{ inputs.grpcio_version }}
       run: poetry add grpcio==${{ inputs.grpcio_version }}
       shell: bash
-    
+
     - name: Install lz4 ${{ inputs.lz4_version }}
       run: poetry add lz4==${{ inputs.lz4_version }}
       shell: bash
-    
+
     - name: Install protobuf ${{ inputs.protobuf_version }}
       run: poetry add protobuf==${{ inputs.protobuf_version }}
       shell: bash
-    
+
     - name: Install googleapis-common-protos ${{ inputs.googleapis_common_protos_version }}
       run: poetry add googleapis-common-protos==${{ inputs.googleapis_common_protos_version }}
       shell: bash
-    
+
     - uses: nick-fields/retry@v3
       with:
         timeout_minutes: 5
         max_attempts: 3
         retry_on: error
-        command: poetry run pytest tests/dependency/grpc -s -v
+        command: poetry run pytest tests/dependency/grpc -s -v --ddtrace
       env:
         PINECONE_API_KEY: ${{ inputs.PINECONE_API_KEY }}
-        INDEX_NAME: ${{ inputs.index_name }}  
+        INDEX_NAME: ${{ inputs.index_name }}
diff --git a/.github/actions/test-dependency-rest/action.yaml b/.github/actions/test-dependency-rest/action.yaml
@@ -39,7 +39,7 @@ runs:
         timeout_minutes: 5
         max_attempts: 3
         retry_on: error
-        command: poetry run pytest tests/dependency/rest -s -v
+        command: poetry run pytest tests/dependency/rest -s -v --ddtrace
       env:
         PINECONE_API_KEY: '${{ inputs.PINECONE_API_KEY }}'
-        INDEX_NAME: '${{ inputs.index_name }}'
+        INDEX_NAME: '${{ inputs.index_name }}'
diff --git a/.github/workflows/testing-dependency.yaml b/.github/workflows/testing-dependency.yaml
@@ -3,6 +3,13 @@ name: Dependency Testing
 on:
   workflow_call: {}
 
+env:
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+  DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+  DD_SITE: datadoghq.com
+  DD_ENV: ci
+  DD_SERVICE: pinecone-python-client
+
 jobs:
   dependency-matrix-setup:
     name: Deps setup
@@ -72,6 +79,7 @@ jobs:
           lz4_version: '${{ matrix.lz4_version }}'
           protobuf_version: '${{ matrix.protobuf_version }}'
           googleapis_common_protos_version: '${{ matrix.googleapis_common_protos_version }}'
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.grpcio:${{ matrix.grpcio_version }},test.configuration.lz4:${{ matrix.lz4_version }},test.configuration.protobuf:${{ matrix.protobuf_version }},test.configuration.googleapis_common_protos:${{ matrix.googleapis_common_protos_version }}'
 
   dependency-matrix-grpc-312:
     name: Deps (GRPC)
@@ -107,6 +115,7 @@ jobs:
           lz4_version: '${{ matrix.lz4_version }}'
           protobuf_version: '${{ matrix.protobuf_version }}'
           googleapis_common_protos_version: '${{ matrix.googleapis_common_protos_version }}'
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.grpcio:${{ matrix.grpcio_version }},test.configuration.lz4:${{ matrix.lz4_version }},test.configuration.protobuf:${{ matrix.protobuf_version }},test.configuration.googleapis_common_protos:${{ matrix.googleapis_common_protos_version }}'
 
   dependency-matrix-rest:
     name: Deps (REST)
@@ -132,7 +141,8 @@ jobs:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           urllib3_version: '${{ matrix.urllib3_version }}'
-
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.urllib3:${{ matrix.urllib3_version }}'
+
   dependency-matrix-rest-312:
     name: Deps (REST)
     runs-on: ubuntu-latest
@@ -156,6 +166,7 @@ jobs:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           urllib3_version: '${{ matrix.urllib3_version }}'
+          DD_TAGS: 'test.configuration.python:${{ matrix.python_version}},test.configuration.urllib3:${{ matrix.urllib3_version }}'
 
   deps-cleanup:
     name: Deps cleanup
@@ -171,4 +182,4 @@ jobs:
       - uses: ./.github/actions/delete-index
         with:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
-          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
+          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
diff --git a/.github/workflows/testing-integration.yaml b/.github/workflows/testing-integration.yaml
@@ -2,6 +2,13 @@ name: "Integration Tests"
 'on':
   workflow_call: {}
 
+env:
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+  DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+  DD_SITE: datadoghq.com
+  DD_ENV: ci
+  DD_SERVICE: pinecone-python-client
+
 jobs:
   plugin-inference:
     name: Test inference plugin
@@ -20,7 +27,7 @@ jobs:
         with:
           include_grpc: 'true'
       - name: 'Run integration tests'
-        run: poetry run pytest tests/integration/inference -s -vv
+        run: poetry run pytest tests/integration/inference -s -vv --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
@@ -33,7 +40,7 @@ jobs:
       matrix:
         python_version: [3.8, 3.12]
         use_grpc: [true, false]
-        metric: 
+        metric:
           - cosine
           # - euclidean
           # - dotproduct
@@ -43,6 +50,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test-data-plane
         with:
+          DATADOG_API_KEY: '${{ secrets.DATADOG_API_KEY }}'
           python_version: '${{ matrix.python_version }}'
           use_grpc: '${{ matrix.use_grpc }}'
           metric: '${{ matrix.metric }}'
@@ -91,7 +99,7 @@ jobs:
         uses: ./.github/actions/setup-poetry
       - name: 'Run integration tests (REST, prod)'
         if: matrix.pineconeEnv == 'prod'
-        run: poetry run pytest tests/integration/control/pod -s -v
+        run: poetry run pytest tests/integration/control/pod -s -v --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
@@ -101,7 +109,7 @@ jobs:
           METRIC: 'cosine'
       - name: 'Run integration tests (REST, staging)'
         if: matrix.pineconeEnv == 'staging'
-        run: poetry run pytest tests/integration/control/pod -s -v
+        run: poetry run pytest tests/integration/control/pod -s -v --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_CONTROLLER_HOST: 'https://api-staging.pinecone.io'
@@ -136,7 +144,7 @@ jobs:
         uses: ./.github/actions/setup-poetry
       - name: 'Run integration tests (REST, prod)'
         if: matrix.pineconeEnv == 'prod'
-        run: poetry run pytest tests/integration/control/serverless -s -vv
+        run: poetry run pytest tests/integration/control/serverless -s -vv --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
@@ -145,7 +153,7 @@ jobs:
           SERVERLESS_REGION: '${{ matrix.testConfig.serverless.region }}'
       - name: 'Run integration tests (REST, staging)'
         if: matrix.pineconeEnv == 'staging'
-        run: poetry run pytest tests/integration/control/serverless -s -vv
+        run: poetry run pytest tests/integration/control/serverless -s -vv --ddtrace
         env:
           PINECONE_DEBUG_CURL: 'true'
           PINECONE_CONTROLLER_HOST: 'https://api-staging.pinecone.io'

diff --git a/.github/workflows/testing-unit.yaml b/.github/workflows/testing-unit.yaml
@@ -2,6 +2,13 @@ name: "Unit Tests"
 'on':
   workflow_call: {}
 
+env:
+  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+  DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+  DD_SITE: datadoghq.com
+  DD_ENV: ci
+  DD_SERVICE: pinecone-python-client
+
 jobs:
   unit-tests:
     name: Unit tests
@@ -30,10 +37,10 @@ jobs:
           include_grpc: '${{ matrix.use_grpc }}'
           include_types: true
       - name: Run unit tests (REST)
-        run: poetry run pytest --cov=pinecone --timeout=120 tests/unit
+        run: poetry run pytest --cov=pinecone --timeout=120 tests/unit --ddtrace
       - name: Run unit tests (GRPC)
         if: ${{ matrix.use_grpc == true }}
-        run: poetry run pytest --cov=pinecone/grpc --timeout=120 tests/unit_grpc
+        run: poetry run pytest --cov=pinecone/grpc --timeout=120 tests/unit_grpc --ddtrace
       - name: mypy check
         env:
           INCLUDE_GRPC: '${{ matrix.use_grpc }}'