PTFE-782 Add runner group synchronisation method (#438)

Co-authored-by: Thomas Carmet <[email protected]>

.trunk/trunk.yaml

@@ -23,9 +23,9 @@ lint:
     - [email protected]
     - [email protected]
     - [email protected]
-    - black@23.7.0
+    - black@23.9.1
     - [email protected]
-    - [email protected].287
+    - [email protected].288
     - [email protected]
     - [email protected]
     - [email protected]
@@ -43,7 +43,7 @@ lint:
     - terrascan
 runtimes:
   enabled:
-    - go@1.19.5
+    - go@1.21.0
     - [email protected]
     - python@>=3.11.4
 actions:

runner_manager/jobs/startup.py

@@ -17,19 +17,26 @@
 log = logging.getLogger(__name__)
 
 
-def create_runner_groups(settings: Settings, github: GitHub):
+def sync_runner_groups(settings: Settings, github: GitHub):
+    """Sync runner groups between the settings of the database and GitHub.
+
+    Args:
+        settings (Settings): Settings of the application
+        github (GitHub): GitHub client
+    """
+
     runner_groups_configs = settings.runner_groups
     existing_groups: List[RunnerGroup] = RunnerGroup.find().all()
-
     for runner_group_config in runner_groups_configs:
-        if runner_group_config.name not in [group.name for group in existing_groups]:
-            runner_group: RunnerGroup = RunnerGroup(**runner_group_config.dict())
+        if runner_group_config.name in [group.name for group in existing_groups]:
+            runner_group: RunnerGroup = RunnerGroup.find_from_base(runner_group_config)
+            runner_group.update(**runner_group_config.dict())
             runner_group.save(github=github)
+            existing_groups.remove(runner_group)
         else:
-            runner_group: RunnerGroup = RunnerGroup.find_from_base(runner_group_config)
+            runner_group: RunnerGroup = RunnerGroup(**runner_group_config.dict())
+            runner_group.save(github=github)
 
-            existing_groups.remove(runner_group)
-            runner_group.update(**runner_group_config.dict())
     for runner_group in existing_groups:
         log.info(f"Deleting runner group {runner_group.name}")
         runner_group.delete(pk=runner_group.pk, github=github)
@@ -79,7 +86,7 @@ def startup(settings: Settings = get_settings()):
     github: GitHub = get_github()
     Migrator().run()
     log.info("Creating runner groups...")
-    create_runner_groups(settings, github)
+    sync_runner_groups(settings, github)
     log.info("Runner groups created.")
     log.info("Bootstrapping healthchecks...")
     bootstrap_healthchecks(settings)

runner_manager/models/runner_group.py

@@ -190,6 +190,22 @@ def delete_runner(self, runner: Runner, github: GitHub) -> int:
                 )
         return self.backend.delete(runner)
 
+    def find_github_group(self, github: GitHub) -> GitHubRunnerGroup | None:
+        """
+        List the groups defined in GitHub and return a
+        GitHubRunnerGroup instance if the group exists, None otherwise.
+
+        A group is considered to exist if the name matches.
+        """
+        for group in github.paginate(
+            github.rest.actions.list_self_hosted_runner_groups_for_org,
+            map_func=lambda r: r.parsed_data.runner_groups,
+            org=self.organization,
+        ):
+            group: GitHubRunnerGroup
+            if group.name == self.name:
+                return group
+
     @property
     def need_new_runner(self) -> bool:
         runners = self.get_runners()
@@ -200,6 +216,10 @@ def need_new_runner(self) -> bool:
     def create_github_group(self, github: GitHub) -> GitHubRunnerGroup:
         """Create a GitHub runner group."""
 
+        if self.id is None:
+            exists = self.find_github_group(github)
+            self.id = exists.id if exists is not None else None
+
         group: Response[GitHubRunnerGroup]
         data = GitHubRunnerGroup(
             name=self.name,

tests/unit/conftest.py

@@ -1,8 +1,11 @@
 from base64 import b64encode
 from datetime import datetime, timedelta, timezone
+from typing import List, TypeVar, cast
 
 import httpx
 from githubkit.config import Config
+from githubkit.paginator import Paginator
+from githubkit.response import Response
 from hypothesis import HealthCheck
 from hypothesis import settings as hypothesis_settings
 from pytest import fixture
@@ -20,8 +23,32 @@
 hypothesis_settings.load_profile("unit")
 
 
+RT = TypeVar("RT")
+
+
+def get_next_monkeypatch(self: Paginator):
+    """monkeypatch Paginator._get_next_page to limit the number of pages to 2"""
+    if self._current_page == 2:
+        return []
+
+    response = self.request(
+        *self.args,
+        **self.kwargs,
+        page=self._current_page,  # type: ignore
+        per_page=self._per_page,  # type: ignore
+    )
+    self._cached_data = (
+        cast(Response[List[RT]], response).parsed_data
+        if self.map_func is None
+        else self.map_func(response)
+    )
+    self._index = 0
+    self._current_page += 1
+    return self._cached_data
+
+
 @fixture()
-def github(settings) -> GitHub:
+def github(settings, monkeypatch) -> GitHub:
     """
     Return a GitHub client configured with:
 
@@ -38,6 +65,7 @@ def github(settings) -> GitHub:
         timeout=httpx.Timeout(5.0),
     )
 
+    monkeypatch.setattr(Paginator, "_get_next_page", get_next_monkeypatch)
     return GitHub(config=config)
 
 

tests/unit/models/test_runner_group.py

@@ -206,3 +206,11 @@ def test_runner_group_queue(runner_group: RunnerGroup, github: GitHub):
     runner4 = runner_group.create_runner(github)
     assert runner4 is not None
     assert runner_group.queued == 0
+
+
+def test_find_github_group(runner_group: RunnerGroup, github: GitHub):
+    runner_group.name = "octo-runner-group"
+    exists = runner_group.find_github_group(github)
+    assert exists is not None
+    group = runner_group.save(github=github)
+    assert exists.id == group.id