Merge pull request #39 from pinecone-io/ci_system_tests

Ci system tests

.github/workflows/CI.yml

@@ -46,11 +46,17 @@ jobs:
         version: 1.3.2
     - name: install dependencies
       run: poetry install --with dev --all-extras
-    - name: Run pytest
-      run: poetry run pytest -v --html=report.html tests/unit
+    - name: Run unit tests
+      run: poetry run pytest --html=report.html --self-contained-html tests/unit
+    - name: Run system tests
+      env:
+        PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        PINECONE_ENVIRONMENT: eu-west1-gcp
+      run: poetry run pytest -n 3 --dist loadgroup --html=report_system.html --self-contained-html tests/system
     - name: upload pytest report.html
       uses: actions/upload-artifact@v3
       if: always()
       with:
         name: dataset-pytest-report-py${{ matrix.python-version }}
-        path: report.html
+        path: report*.html

pyproject.toml

@@ -18,6 +18,7 @@ pinecone-text = "^0.5.3"
 flake8-pyproject = "^1.2.3"
 pandas-stubs = "^2.0.3.230814"
 langchain = "^0.0.188"
+pytest-xdist = "^3.3.1"
 
 [tool.poetry.group.dev.dependencies]
 jupyter = "^1.0.0"

tests/system/knowledge_base/test_knowledge_base.py

@@ -4,6 +4,7 @@
 import pandas as pd
 from dotenv import load_dotenv
 from context_engine.knoweldge_base import KnowledgeBase
+from context_engine.knoweldge_base.knowledge_base import INDEX_NAME_PREFIX
 from context_engine.knoweldge_base.models import DocumentWithScore
 from context_engine.models.data_models import Document, Query
 from tests.unit.stubs.stub_record_encoder import StubRecordEncoder
@@ -13,6 +14,7 @@
 load_dotenv()
 
 
+@pytest.mark.xdist_group(name="group1")
 class TestKnowledgeBase:
 
     @staticmethod
@@ -28,8 +30,8 @@ def encoder():
 
     @staticmethod
     @pytest.fixture(scope="class", autouse=True)
-    def knowledge_base(chunker, encoder):
-        kb = KnowledgeBase(index_name_suffix="kb-integration-test",
+    def knowledge_base(testrun_uid, chunker, encoder):
+        kb = KnowledgeBase(index_name_suffix=f"kb-integration-test-{testrun_uid[-6:]}",
                            encoder=encoder,
                            chunker=chunker)
         pinecone.init()
@@ -69,7 +71,6 @@ def encoded_chunks(documents, chunker, encoder):
     def test_create_index(knowledge_base):
         index_name = knowledge_base._index_name
         assert index_name in pinecone.list_indexes()
-        assert index_name == "context-engine-kb-integration-test"
         assert knowledge_base._index.describe_index_stats()
 
     @staticmethod
@@ -83,8 +84,16 @@ def test_connect_force_connected_kb(knowledge_base):
         assert knowledge_base._index.describe_index_stats()
 
     @staticmethod
-    def test_connect_unconnected_kb_index_exist():
-        kb = KnowledgeBase(index_name_suffix="kb-integration-test",
+    def test_connect_unconnected_kb_index_exist(knowledge_base):
+        index_name = knowledge_base._index_name
+        kb = KnowledgeBase(index_name_suffix=index_name,
+                           encoder=StubRecordEncoder(
+                               StubDenseEncoder(dimension=3)),
+                           chunker=StubChunker())
+        kb.connect()
+        assert kb._index.describe_index_stats()
+        stripped_index_name = index_name.rstrip(INDEX_NAME_PREFIX)
+        kb = KnowledgeBase(index_name_suffix=stripped_index_name,
                            encoder=StubRecordEncoder(
                                StubDenseEncoder(dimension=3)),
                            chunker=StubChunker())
@@ -173,10 +182,12 @@ def test_delete_documents(knowledge_base, encoded_chunks):
         TestKnowledgeBase.assert_ids_not_in_index(knowledge_base, chunk_ids)
 
     @staticmethod
-    def test_update_documents(encoder, documents, encoded_chunks):
+    def test_update_documents(encoder, documents, encoded_chunks, knowledge_base):
+        index_name = knowledge_base._index_name
+
         # chunker/kb that produces less chunks per doc
         chunker = StubChunker(num_chunks_per_doc=1)
-        kb = KnowledgeBase(index_name_suffix="kb-integration-test",
+        kb = KnowledgeBase(index_name_suffix=index_name,
                            encoder=encoder,
                            chunker=chunker)
         kb.connect()