Merge pull request #126 from confident-ai/feature/add-chatbot-tset

add chatbot test

.github/workflows/test-no-login.yml

@@ -0,0 +1,47 @@
+name: Run pytest in matrix
+
+on: push
+
+jobs:
+  examples:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          # cache: "pip"
+
+      # - name: Get pip cache dir
+      #   id: pip-cache
+      #   run: echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache dependencies
+        id: cache-dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: dependencies-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            dependencies-${{ matrix.python-version }}-
+            dependencies-
+
+      - name: Install dependencies
+        env:
+          python-version: ${{ matrix.python-version }}
+        run: |
+          python -c "import sys; print(sys.version)"
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt ragas
+          python -m pip install . pytest-rerunfailures pytest-asyncio
+
+      - name: Run Unit Tests (pytest)
+        env:
+          python-version: ${{ matrix.python-version }}
+        run: |
+          deepeval test run -x tests

deepeval/__init__.py

@@ -9,4 +9,23 @@
 #     test_utils,
 #     utils,
 # )
+import warnings
 from ._version import __version__
+
+
+def check_for_update():
+    try:
+        import requests
+
+        response = requests.get("https://pypi.org/pypi/deepeval/json")
+        latest_version = response.json()["info"]["version"]
+        if latest_version > __version__:
+            warnings.warn(
+                f'You are using deepeval version {__version__}, however version {latest_version} is available. You should consider upgrading via the "pip install --upgrade deepeval" command.'
+            )
+    except ModuleNotFoundError:
+        # they're just getting the version
+        pass
+
+
+check_for_update()

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "0.17.2"
+__version__: str = "0.17.3"

deepeval/api.py

@@ -54,6 +54,13 @@ def from_metric(cls, metric: Metric):
         return cls(metric=metric.__name__, score=metric.score)
 
 
+class TestRunResponse(BaseModel):
+    """Add Test Run Results"""
+
+    testRunId: str
+    projectId: str
+
+
 class TestRun(BaseModel):
     test_file: Optional[str] = Field(
         # TODO: Fix test_file
@@ -466,15 +473,19 @@ def list_implementations(self):
         """
         return self.get_request(endpoint="/v1/implementation")
 
-    def post_test_run(self, test_run: TestRun):
+    def post_test_run(self, test_run: TestRun) -> TestRunResponse:
         """Post a test run"""
         try:
             body = test_run.model_dump(by_alias=True)
         except AttributeError:
             # Pydantic version below 2.0
             body = test_run.dict(by_alias=True)
 
-        return self.post_request(
+        result = self.post_request(
             endpoint="/v1/test-run",
             body=body,
         )
+        response = TestRunResponse(
+            testRunId=result["testRunId"], projectId=result["projectId"]
+        )
+        return response

deepeval/plugins/plugin.py

@@ -1,5 +1,6 @@
 import pytest
 import os
+from rich import print
 from deepeval.api import Api, TestRun
 
 from deepeval.constants import PYTEST_RUN_ENV_VAR
@@ -28,9 +29,8 @@ def pytest_sessionfinish(session, exitstatus):
         api: Api = Api()
         test_run = TestRun.load(test_filename)
         result = api.post_test_run(test_run)
-        run_id = result["id"]
+        link = f"https://app.confident-ai.com/project/{result.projectId}/unit-tests/{result.testRunId}"
         print(
-            "✅ Tests finished! View results on "
-            + f"https://app.confident-ai.com/unit-tests/{run_id}"
+            "✅ Tests finished! View results on " f"[link={link}]{link}[/link]"
         )
         os.remove(test_filename)

tests/test_chatbot_example.py

@@ -0,0 +1,23 @@
+import pytest
+from deepeval.metrics.factual_consistency import FactualConsistencyMetric
+from deepeval.test_case import LLMTestCase
+from deepeval.run_test import assert_test
+
+
+def test_1():
+    input = "What does your company do?"
+    output = "My company doesn't do anything."
+    context = "Our company specializes in cloud computing, data analytics, and machine learning. We offer a range of services including cloud storage solutions, data analytics platforms, and custom machine learning models."
+    factual_consistency_metric = FactualConsistencyMetric(minimum_score=1.0)
+    test_case = LLMTestCase(output=output, context=context)
+    with pytest.raises(AssertionError):
+        assert_test(test_case, [factual_consistency_metric])
+
+
+def test_2():
+    input = "What does your company do?"
+    output = "My company is a cloud computing company."
+    context = "Our company specializes in cloud computing, data analytics, and machine learning. We offer a range of services including cloud storage solutions, data analytics platforms, and custom machine learning models."
+    factual_consistency_metric = FactualConsistencyMetric(minimum_score=0.5)
+    test_case = LLMTestCase(output=output, context=context)
+    assert_test(test_case, [factual_consistency_metric])