Merge pull request #110 from lsst-sqre/tickets/DM-33244

[DM-33244] Convert to new moneypenny API

pyproject.toml

@@ -93,3 +93,10 @@ include_trailing_comma = true
 multi_line_output = 3
 known_first_party = ["nublado2", "tests"]
 skip = ["docs/conf.py"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "strict"
+python_files = [
+    "tests/*.py",
+    "tests/*/*.py"
+]

src/nublado2/provisioner.py

@@ -2,18 +2,16 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
 from urllib.parse import urljoin
 
-from jupyterhub.utils import exponential_backoff
+from aiohttp import ClientTimeout
+from jupyterhub.spawner import Spawner
+from tornado import web
 from traitlets.config import LoggingConfigurable
 
 from nublado2.http import get_session
 from nublado2.nublado_config import NubladoConfig
 
-if TYPE_CHECKING:
-    from jupyterhub.spawner import Spawner
-
 __all__ = ["Provisioner"]
 
 
@@ -42,7 +40,7 @@ async def provision_homedir(self, spawner: Spawner) -> None:
             "uid": int(auth_state["uid"]),
             "groups": auth_state["groups"],
         }
-        provision_url = urljoin(base_url, "moneypenny/commission")
+        provision_url = urljoin(base_url, "moneypenny/users")
         session = await get_session()
         self.log.debug(f"Posting dossier {dossier} to {provision_url}")
         r = await session.post(
@@ -53,36 +51,28 @@ async def provision_homedir(self, spawner: Spawner) -> None:
         self.log.debug(f"POST got {r.status}")
         r.raise_for_status()
 
-        # Use a wrapper to log the number of requests.
-        count = 0
-
-        async def _wait_wrapper() -> bool:
-            nonlocal count
-            count += 1
-            self.log.debug(f"Checking Moneypenny status #{count}")
+        # Wait until the work has finished.
+        data = await r.json()
+        if data["status"] != "active":
             return await self._wait_for_provision(spawner.user.name)
 
-        # Run with exponential backoff and a maximum timeout of 5m.
-        await exponential_backoff(
-            _wait_wrapper,
-            fail_message="Moneypenny did not complete",
-            timeout=300,
-        )
-
-    async def _wait_for_provision(self, username: str) -> bool:
+    async def _wait_for_provision(self, username: str) -> None:
         """Wait for provisioning to complete."""
         base_url = self.nublado_config.base_url
-        status_url = urljoin(base_url, f"moneypenny/{username}")
+        status_url = urljoin(base_url, f"moneypenny/users/{username}/wait")
         token = self.nublado_config.gafaelfawr_token
         session = await get_session()
 
         r = await session.get(
-            status_url, headers={"Authorization": f"Bearer {token}"}
+            status_url,
+            headers={"Authorization": f"Bearer {token}"},
+            timeout=ClientTimeout(total=300),
         )
         self.log.debug(f"Moneypenny {status_url} status: {r.status}")
-
-        if r.status == 200 or r.status == 404:
-            return True
-        if r.status != 202:
+        if r.status == 200:
+            data = await r.json()
+            if data["status"] != "active":
+                status = data["status"]
+                raise web.HTTPError(500, f"Moneypenny reports status {status}")
+        else:
             r.raise_for_status()
-        return False

tests/auth_test.py

@@ -7,10 +7,11 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import Any, AsyncGenerator, Callable, Dict
 from unittest.mock import MagicMock, patch
 
 import pytest
+import pytest_asyncio
 from aioresponses import CallbackResult, aioresponses
 from tornado import web
 from tornado.httputil import HTTPHeaders
@@ -21,11 +22,8 @@
     _build_auth_info,
 )
 
-if TYPE_CHECKING:
-    from typing import Any, AsyncGenerator, Callable, Dict
 
-
-@pytest.fixture(autouse=True)
+@pytest_asyncio.fixture(autouse=True)
 async def config_mock() -> AsyncGenerator:
     """Use a mock NubladoConfig object."""
     with patch("nublado2.auth.NubladoConfig") as mock:

tests/provisioner_test.py

@@ -2,23 +2,20 @@
 
 from __future__ import annotations
 
-import asyncio
-import sys
-from typing import TYPE_CHECKING
+import json
+from typing import Any, AsyncGenerator, Callable, Dict, List, Union
 from unittest.mock import MagicMock, Mock, patch
 
 import pytest
+import pytest_asyncio
 from aioresponses import CallbackResult, aioresponses
 from jupyterhub.spawner import Spawner
 from jupyterhub.user import User
 
 from nublado2.resourcemgr import ResourceManager
 
-if TYPE_CHECKING:
-    from typing import Any, AsyncGenerator, Callable, Dict, List, Union
 
-
-@pytest.fixture(autouse=True)
+@pytest_asyncio.fixture(autouse=True)
 async def config_mock() -> AsyncGenerator:
     """Use a mock NubladoConfig object."""
     with patch("nublado2.resourcemgr.NubladoConfig") as mock:
@@ -35,28 +32,30 @@ def build_handler(
     username: str,
     uid: int,
     groups: List[Dict[str, Union[str, int]]],
-    *,
-    count: int,
 ) -> Callable[..., CallbackResult]:
     probe = 0
 
     def handler(url: str, **kwargs: Any) -> CallbackResult:
-        if str(url) == "https://data.example.com/moneypenny/commission":
+        nonlocal probe
+        user_url = f"https://data.example.com/moneypenny/users/{username}"
+        if str(url) == "https://data.example.com/moneypenny/users":
             assert kwargs["json"] == {
                 "username": username,
                 "uid": uid,
                 "groups": groups,
             }
-            return CallbackResult(status=202)
-        elif str(url) == f"https://data.example.com/moneypenny/{username}":
-            nonlocal probe
+            return CallbackResult(status=303, headers={"Location": user_url})
+        elif str(url) == user_url:
+            result = {
+                "username": username,
+                "status": "commissioning" if probe == 0 else "active",
+                "uid": uid,
+                "groups": groups,
+            }
+            return CallbackResult(status=200, body=json.dumps(result))
+        elif str(url) == f"{user_url}/wait":
             probe += 1
-            if probe == count:
-                return CallbackResult(status=200)
-            elif probe > count:
-                return CallbackResult(status=404)
-            else:
-                return CallbackResult(status=202)
+            return CallbackResult(status=303, headers={"Location": user_url})
         else:
             assert False, f"unknown URL {str(url)}"
 
@@ -73,20 +72,16 @@ async def test_provision() -> None:
         "uid": 1234,
         "groups": [{"name": "foo", "id": 1234}],
     }
+    spawner.user.get_auth_state.return_value = auth_state
 
-    # AsyncMock was introduced in Python 3.8, so sadly we can't use it yet.
-    if sys.version_info < (3, 8):
-        spawner.user.get_auth_state.return_value = asyncio.Future()
-        spawner.user.get_auth_state.return_value.set_result(auth_state)
-    else:
-        spawner.user.get_auth_state.return_value = auth_state
-
-    commission_url = "https://data.example.com/moneypenny/commission"
-    status_url = "https://data.example.com/moneypenny/someuser"
+    commission_url = "https://data.example.com/moneypenny/users"
+    status_url = "https://data.example.com/moneypenny/users/someuser"
+    wait_url = "https://data.example.com/moneypenny/users/someuser/wait"
     with aioresponses() as m:
         handler = build_handler(
-            "someuser", 1234, [{"name": "foo", "id": 1234}], count=2
+            "someuser", 1234, [{"name": "foo", "id": 1234}]
         )
         m.post(commission_url, callback=handler)
         m.get(status_url, callback=handler, repeat=True)
+        m.get(wait_url, callback=handler)
         await resource_manager.provisioner.provision_homedir(spawner)

tests/resourcemgr_test.py

@@ -4,7 +4,7 @@
 
 import asyncio
 import sys
-from typing import TYPE_CHECKING
+from typing import Any, Callable, Dict, Iterator, List
 from unittest.mock import Mock, patch
 
 import pytest
@@ -25,9 +25,6 @@
 from nublado2.resourcemgr import ResourceManager
 from nublado2.selectedoptions import SelectedOptions
 
-if TYPE_CHECKING:
-    from typing import Any, Callable, Dict, Iterator, List
-
 # Mock user resources template to test the template engine.
 USER_RESOURCES_TEMPLATE = """
 - apiVersion: v1