👽️(api) migrate ralph-malph API to 4.0.0

Ralph 4.0.0 has introduced breaking changes in its API. It has to be adapted in
its use in warren-api package.

.env.dist

@@ -44,8 +44,8 @@ POSTGRES_PASSWORD=pass
 
 # Ralph LRS
 RALPH_AUTH_FILE=/app/.ralph/auth.json
-RALPH_BACKENDS__DATABASE__ES__HOSTS=http://elasticsearch:9200
-RALPH_BACKENDS__DATABASE__ES__INDEX=statements
+RALPH_BACKENDS__LRS__ES__HOSTS=http://elasticsearch:9200
+RALPH_BACKENDS__LRS__ES__DEFAULT_INDEX=statements
 RALPH_RUNSERVER_PORT=8200
 
 # Sentry

docker-compose.yml

@@ -83,7 +83,7 @@ services:
 
   # -- backends
   ralph:
-    image: fundocker/ralph:3.8.0
+    image: fundocker/ralph:4.0.0
     user: ${DOCKER_USER:-1000}
     env_file:
       - .env

src/api/core/warren/api/health.py

@@ -1,11 +1,10 @@
 """API routes related to application health checking."""
 
 import logging
-from enum import Enum, unique
 
 from fastapi import APIRouter, Response, status
 from pydantic import BaseModel
-from ralph.backends.http.base import HTTPBackendStatus
+from ralph.backends.data.base import DataBackendStatus
 
 from warren.backends import lrs_client
 from warren.db import is_alive as is_db_alive
@@ -15,25 +14,16 @@
 router = APIRouter()
 
 
-@unique
-class BackendStatus(str, Enum):
-    """Generic backend statuses inspired from Ralph HTTP backend."""
-
-    OK = "ok"
-    AWAY = "away"
-    ERROR = "error"
-
-
 class Heartbeat(BaseModel):
     """Warren backends status."""
 
-    database: BackendStatus
-    lrs: HTTPBackendStatus
+    data: DataBackendStatus
+    lrs: DataBackendStatus
 
     @property
     def is_alive(self):
         """A helper that checks the overall status."""
-        if self.database == BackendStatus.OK and self.lrs == HTTPBackendStatus.OK:
+        if self.data == DataBackendStatus.OK and self.lrs == DataBackendStatus.OK:
             return True
         return False
 
@@ -54,7 +44,7 @@ async def heartbeat(response: Response) -> Heartbeat:
     Return a 200 if all checks are successful.
     """
     statuses = Heartbeat(
-        database=BackendStatus.OK if is_db_alive() else BackendStatus.ERROR,
+        data=DataBackendStatus.OK if is_db_alive() else DataBackendStatus.ERROR,
         lrs=await lrs_client.status(),
     )
     if not statuses.is_alive:

src/api/core/warren/backends.py

@@ -1,15 +1,17 @@
 """Backends for warren."""
 
-from ralph.backends.http import AsyncLRSHTTP
-from ralph.conf import LRSHeaders
+from ralph.backends.data.async_lrs import AsyncLRSDataBackend
+from ralph.backends.data.lrs import LRSDataBackendSettings, LRSHeaders
 
 from warren.conf import settings
 
-lrs_client = AsyncLRSHTTP(
-    base_url=settings.LRS_HOSTS,
-    username=settings.LRS_AUTH_BASIC_USERNAME,
-    password=settings.LRS_AUTH_BASIC_PASSWORD,
-    headers=LRSHeaders(
+lrs_client_settings = LRSDataBackendSettings(
+    BASE_URL=settings.LRS_HOSTS,
+    USERNAME=settings.LRS_AUTH_BASIC_USERNAME,
+    PASSWORD=settings.LRS_AUTH_BASIC_PASSWORD,
+    HEADERS=LRSHeaders(
         X_EXPERIENCE_API_VERSION="1.0.3", CONTENT_TYPE="application/json"
     ),
 )
+
+lrs_client = AsyncLRSDataBackend(settings=lrs_client_settings)

src/api/core/warren/cli.py

@@ -160,7 +160,7 @@ def indicator_compute(ctx: click.Context, indicator: str, cache: bool):
 
         # Cast value given parameter annotation
         if issubclass(parameter.annotation, str):
-            pass
+            value = value.strip('"')
         elif issubclass(parameter.annotation, (dict, list)):
             value = json.loads(value)
         elif issubclass(parameter.annotation, BaseModel):

src/api/core/warren/indicators/base.py

@@ -5,12 +5,13 @@
 from functools import cached_property
 from typing import List, Optional
 
+from ralph.backends.lrs.base import LRSStatementsQuery
 from ralph.exceptions import BackendException
 
 from warren.backends import lrs_client as async_lrs_client
 from warren.exceptions import LrsClientException
 from warren.filters import Datetime, DatetimeRange
-from warren.models import XAPI_STATEMENT, LRSStatementsQuery
+from warren.models import XAPI_STATEMENT
 
 
 class BaseIndicator(ABC):
@@ -96,7 +97,7 @@ async def fetch_statements(self) -> List[XAPI_STATEMENT]:
             return [
                 value
                 async for value in self.lrs_client.read(
-                    target=self.lrs_client.statements_endpoint,
+                    target=self.lrs_client.settings.STATEMENTS_ENDPOINT,
                     query=self.get_lrs_query(),
                 )
             ]

src/api/core/warren/indicators/mixins.py

@@ -72,7 +72,9 @@ def cache_key(self) -> str:
 
         """
         lrs_query_hash = hashlib.sha256(
-            self.get_lrs_query().json(exclude={"query": {"since", "until"}}).encode()
+            self.get_lrs_query()
+            .json(exclude={"since", "until"}, exclude_none=True, exclude_unset=True)
+            .encode()
         ).hexdigest()
         return f"{self.__class__.__name__.lower()}-{lrs_query_hash}"
 

src/api/core/warren/models.py

@@ -6,9 +6,7 @@
 
 import arrow
 from lti_toolbox.launch_params import LTIRole
-from pydantic.dataclasses import dataclass as pdt_dataclass
 from pydantic.main import BaseModel
-from ralph.backends.database.base import StatementParameters
 
 from warren.fields import Date
 
@@ -238,14 +236,6 @@ def to_daily_counts(self):
         total = sum(c.count for c in counts)
         return DailyCounts(total=total, counts=counts)
 
-
-# FIXME: prefer using a valid generic pydantic model, this is too convoluted.
-# See: https://github.com/openfun/ralph/issues/425
-# Get a pydantic model from a stdlib dataclass to use Pydantic helpers
-# Fix mypy errors 'is not valid as type' as soon we use a valid generic pydantic model
-LRSStatementsQuery = pdt_dataclass(StatementParameters)
-
-
 class LTIUser(BaseModel):
     """Model to represent LTI user data."""
 

src/api/core/warren/tests/api/test_health.py

@@ -1,6 +1,6 @@
 """Tests for the health check endpoints."""
 import pytest
-from ralph.backends.http.base import HTTPBackendStatus
+from ralph.backends.data.base import DataBackendStatus
 
 from warren.api import health
 from warren.backends import lrs_client
@@ -19,43 +19,43 @@ async def test_api_health_heartbeat(http_client, monkeypatch):
     """Test the heartbeat healthcheck."""
 
     async def lrs_ok():
-        return HTTPBackendStatus.OK
+        return DataBackendStatus.OK
 
     async def lrs_away():
-        return HTTPBackendStatus.AWAY
+        return DataBackendStatus.AWAY
 
     async def lrs_error():
-        return HTTPBackendStatus.ERROR
+        return DataBackendStatus.ERROR
 
     with monkeypatch.context() as lrs_context:
         lrs_context.setattr(lrs_client, "status", lrs_ok)
         response = await http_client.get("/__heartbeat__")
         assert response.status_code == 200
-        assert response.json() == {"database": "ok", "lrs": "ok"}
+        assert response.json() == {"data": "ok", "lrs": "ok"}
 
         lrs_context.setattr(lrs_client, "status", lrs_away)
         response = await http_client.get("/__heartbeat__")
-        assert response.json() == {"database": "ok", "lrs": "away"}
+        assert response.json() == {"data": "ok", "lrs": "away"}
         assert response.status_code == 500
 
         lrs_context.setattr(lrs_client, "status", lrs_error)
         response = await http_client.get("/__heartbeat__")
-        assert response.json() == {"database": "ok", "lrs": "error"}
+        assert response.json() == {"data": "ok", "lrs": "error"}
         assert response.status_code == 500
 
     with monkeypatch.context() as db_context:
         lrs_context.setattr(lrs_client, "status", lrs_ok)
         db_context.setattr(health, "is_db_alive", lambda: False)
         response = await http_client.get("/__heartbeat__")
-        assert response.json() == {"database": "error", "lrs": "ok"}
+        assert response.json() == {"data": "error", "lrs": "ok"}
         assert response.status_code == 500
 
         db_context.setattr(lrs_client, "status", lrs_away)
         response = await http_client.get("/__heartbeat__")
-        assert response.json() == {"database": "error", "lrs": "away"}
+        assert response.json() == {"data": "error", "lrs": "away"}
         assert response.status_code == 500
 
         db_context.setattr(lrs_client, "status", lrs_error)
         response = await http_client.get("/__heartbeat__")
-        assert response.json() == {"database": "error", "lrs": "error"}
+        assert response.json() == {"data": "error", "lrs": "error"}
         assert response.status_code == 500

src/api/core/warren/tests/indicators/test_base.py

@@ -1,7 +1,7 @@
 """Test the functions from the BaseIndicator class."""
 import pytest
 from pytest_httpx import HTTPXMock
-from ralph.backends.http.async_lrs import LRSQuery
+from ralph.backends.lrs.base import LRSStatementsQuery
 
 from warren.exceptions import LrsClientException
 from warren.indicators.base import BaseIndicator
@@ -18,7 +18,7 @@ async def test_base_indicator_fetch_statements_with_default_query(
 
     class MyIndicator(BaseIndicator):
         def get_lrs_query(self):
-            return LRSQuery(query={"verb": "played"})
+            return LRSStatementsQuery(verb="https://w3id.org/xapi/video/verbs/played")
 
         async def compute(self):
             return None
@@ -28,7 +28,7 @@ async def compute(self):
 
     # Mock the LRS call so that it returns the fixture statements
     httpx_mock.add_response(
-        url="http://fake-lrs.com/xAPI/statements?verb=played&limit=500",
+        url="http://fake-lrs.com/xAPI/statements?verb=https://w3id.org/xapi/video/verbs/played&limit=500",
         method="GET",
         json={"statements": [{"id": 1}, {"id": 2}]},
         status_code=200,
@@ -49,7 +49,7 @@ async def test_base_indicator_fetch_statements_with_lrs_failure(
 
     class MyIndicator(BaseIndicator):
         def get_lrs_query(self):
-            return LRSQuery(query={"verb": "played"})
+            return LRSStatementsQuery(verb="https://w3id.org/xapi/video/verbs/played")
 
         async def compute(self):
             return None
@@ -59,7 +59,7 @@ async def compute(self):
 
     # Mock the LRS call so that it returns fails
     httpx_mock.add_response(
-        url="http://fake-lrs.com/xAPI/statements?verb=played&limit=500",
+        url="http://fake-lrs.com/xAPI/statements?verb=https://w3id.org/xapi/video/verbs/played&limit=500",
         method="GET",
         status_code=500,
     )