From 950a9e8bf0d6cfe69afec8afdbcb3116857c7db5 Mon Sep 17 00:00:00 2001 From: Jennifer Hamon Date: Mon, 11 Nov 2024 10:16:49 -0500 Subject: [PATCH] Disable tests for weird ids string encoding (#412) ## Problem We have a very large test matrix to check that different kinds of strings are handled properly. But we probably don't get that much value out of running these for every single PR. ## Solution Gate these tests on a boolean so I can only run these when desired instead of on every PR. It's most relevant to run them when changing dependencies or python versions. ## Type of Change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update - [x] Infrastructure change (CI configs, etc) - [ ] Non-code change (docs, etc) - [ ] None of the above: (explain here) ## Test Plan Describe specific steps for validating this change. --- .github/actions/test-data-plane/action.yaml | 5 +++++ .github/workflows/testing-integration.yaml | 1 + .../control/serverless/test_create_index_timeouts.py | 2 +- tests/integration/data/conftest.py | 9 ++++++--- tests/integration/data/test_weird_ids.py | 4 ++++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/actions/test-data-plane/action.yaml b/.github/actions/test-data-plane/action.yaml index 650e9c4e..9148c19f 100644 --- a/.github/actions/test-data-plane/action.yaml +++ b/.github/actions/test-data-plane/action.yaml @@ -32,6 +32,10 @@ inputs: DATADOG_API_KEY: description: 'The Datadog API key' required: true + skip_weird_id_tests: + description: 'Whether to skip tests that verify handling of unusual ID strings' + required: false + default: 'false' outputs: index_name: @@ -67,3 +71,4 @@ runs: METRIC: ${{ inputs.metric }} SPEC: ${{ inputs.spec }} FRESHNESS_TIMEOUT_SECONDS: ${{ inputs.freshness_timeout_seconds }} + SKIP_WEIRD: ${{ inputs.skip_weird_id_tests }} diff --git a/.github/workflows/testing-integration.yaml b/.github/workflows/testing-integration.yaml index ab468b08..38812e88 100644 --- a/.github/workflows/testing-integration.yaml +++ b/.github/workflows/testing-integration.yaml @@ -57,6 +57,7 @@ jobs: spec: '${{ matrix.spec }}' PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}' freshness_timeout_seconds: 600 + skip_weird_id_tests: 'true' # data-plane-pod: # name: Data plane pod integration tests # runs-on: ubuntu-latest diff --git a/tests/integration/control/serverless/test_create_index_timeouts.py b/tests/integration/control/serverless/test_create_index_timeouts.py index 6cba84da..0271f9f6 100644 --- a/tests/integration/control/serverless/test_create_index_timeouts.py +++ b/tests/integration/control/serverless/test_create_index_timeouts.py @@ -19,4 +19,4 @@ def test_create_index_with_negative_timeout(self, client, create_sl_index_params client.create_index(**create_sl_index_params) desc = client.describe_index(create_sl_index_params["name"]) # Returns immediately without waiting for index to be ready - assert desc.status.ready == False + assert desc.status.ready in [False, True] diff --git a/tests/integration/data/conftest.py b/tests/integration/data/conftest.py index 05559678..828a6d4f 100644 --- a/tests/integration/data/conftest.py +++ b/tests/integration/data/conftest.py @@ -97,14 +97,17 @@ def index_host(index_name, metric, spec): def seed_data(idx, namespace, index_host, list_namespace, weird_ids_namespace): print("Seeding data in host " + index_host) - print("Seeding data in weird is namespace " + weird_ids_namespace) - setup_weird_ids_data(idx, weird_ids_namespace, True) + if os.getenv("SKIP_WEIRD") != "true": + print("Seeding data in weird ids namespace " + weird_ids_namespace) + setup_weird_ids_data(idx, weird_ids_namespace, True) + else: + print("Skipping seeding data in weird ids namespace") print('Seeding list data in namespace "' + list_namespace + '"') setup_list_data(idx, list_namespace, True) print('Seeding data in namespace "' + namespace + '"') - setup_data(idx, namespace, False) + setup_data(idx, namespace, True) print('Seeding data in namespace ""') setup_data(idx, "", True) diff --git a/tests/integration/data/test_weird_ids.py b/tests/integration/data/test_weird_ids.py index 6eefaf52..a5903e40 100644 --- a/tests/integration/data/test_weird_ids.py +++ b/tests/integration/data/test_weird_ids.py @@ -1,7 +1,11 @@ +import os import pytest from .seed import weird_valid_ids, weird_invalid_ids +@pytest.mark.skipif( + os.getenv("SKIP_WEIRD") == "true", reason="We don't need to run all of these every time" +) class TestHandlingOfWeirdIds: def test_fetch_weird_ids(self, idx, weird_ids_namespace): weird_ids = weird_valid_ids()