Implementd wb settings generation on GV side (#143)

* Added wb config on GV side, added config implementation for s3
google_drive box github

* added dropbox support

* fixed tests

addon_imps/storage/box_dot_com.py

@@ -24,6 +24,12 @@ async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResu
             total_count=1,
         )
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        return {
+            "folder": root_folder_id.split(":")[1],
+            "service": service_name,
+        }
+
     async def get_item_info(self, item_id: str) -> storage.ItemResult:
         async with self.network.GET(
             _box_item_url(item_id),

addon_imps/storage/dropbox.py

@@ -16,6 +16,11 @@ async def get_external_account_id(self, _: dict[str, str]) -> str:
     async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResult:
         return await self.list_child_items(item_id="", page_cursor=page_cursor)
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        return {
+            "folder": root_folder_id,
+        }
+
     async def get_item_info(self, item_id: str) -> storage.ItemResult:
         if not item_id:
             return storage.ItemResult(

addon_imps/storage/figshare.py

@@ -13,9 +13,9 @@
 )
 
 
-FILE_REGEX = re.compile(r"^articles/(?P<article_id>\d*)/files/(?P<file_id>\d*)$")
-ARTICLE_REGEX = re.compile(r"^articles/(?P<article_id>\d*)$")
-PROJECT_REGEX = re.compile(r"^projects/(?P<project_id>\d*)$")
+FILE_REGEX = re.compile(r"^article/(?P<article_id>\d*)/files/(?P<file_id>\d*)$")
+ARTICLE_REGEX = re.compile(r"^article/(?P<article_id>\d*)$")
+PROJECT_REGEX = re.compile(r"^project/(?P<project_id>\d*)$")
 
 
 class FigshareStorageImp(storage.StorageAddonHttpRequestorImp):
@@ -39,6 +39,10 @@ async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResu
             next_sample_cursor=str(page_cursor + 1),
         )
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        segments = root_folder_id.split("/")
+        return {"container_type": segments[0], "container_id": segments[1]}
+
     async def get_item_info(self, item_id: str) -> storage.ItemResult:
         if not item_id:
             return ItemResult(item_id="", item_name="", item_type=ItemType.FOLDER)
@@ -153,7 +157,7 @@ class File(ItemResultable):
     @property
     def item_result(self) -> ItemResult:
         return ItemResult(
-            item_id=f"articles{self.article_id}/files/{self.id}",
+            item_id=f"article/{self.article_id}/files/{self.id}",
             item_name=self.name,
             item_type=ItemType.FILE,
         )
@@ -167,7 +171,7 @@ class Project(ItemResultable):
     @property
     def item_result(self) -> ItemResult:
         return ItemResult(
-            item_id=f"projects/{self.id}",
+            item_id=f"project/{self.id}",
             item_name=self.title,
             item_type=ItemType.FOLDER,
         )
@@ -181,7 +185,7 @@ class Article(ItemResultable):
     @property
     def item_result(self) -> ItemResult:
         return ItemResult(
-            item_id=f"articles/{self.id}",
+            item_id=f"article/{self.id}",
             item_name=self.title,
             item_type=ItemType.FOLDER,
         )

addon_imps/storage/github.py

@@ -35,6 +35,13 @@ async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResu
                     items=items, total_count=len(items)
                 ).with_cursor(self._create_offset_cursor(len(items), page_cursor))
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        owner, repo, _ = self._parse_github_item_id(root_folder_id)
+        return {
+            "owner": owner,
+            "repo": repo,
+        }
+
     async def get_item_info(self, item_id: str) -> storage.ItemResult:
         if item_id == "." or not item_id:
             return storage.ItemResult(

addon_imps/storage/gitlab.py

@@ -47,6 +47,8 @@ async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResu
             )
 
     def _get_next_cursor(self, headers):
+        if not headers.get("Link"):
+            return
         next_link_candidates = [
             item for item in headers["Link"].split(",") if 'rel="next"' in item
         ]
@@ -170,13 +172,13 @@ def parse_item(repo_id: str, raw_item: dict) -> ItemResult:
 # module-local helpers
 @dataclass(frozen=True, slots=True)
 class Repository(ItemResultable):
-    id: str
+    path_with_namespace: str
     name: str
 
     @property
     def item_result(self) -> ItemResult:
         return ItemResult(
-            item_id=f"{self.id}:",
+            item_id=f"{quote_plus(self.path_with_namespace)}:",
             item_name=self.name,
             item_type=ItemType.FOLDER,
         )

addon_imps/storage/google_drive.py

@@ -27,6 +27,9 @@ async def get_external_account_id(self, _: dict[str, str]) -> str:
     async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResult:
         return ItemSampleResult(items=[await self.get_item_info("root")], total_count=1)
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        return {"folder": {"id": root_folder_id}}
+
     async def get_item_info(self, item_id: str) -> storage.ItemResult:
         item_id = item_id or "root"
         async with self.network.GET(f"drive/v3/files/{item_id}") as response:

addon_imps/storage/s3.py

@@ -25,6 +25,13 @@ def create_client(credentials: AccessKeySecretKeyCredentials):
             "s3", aws_access_key_id=access_key, aws_secret_access_key=secret_key
         )
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        return {
+            "bucket": root_folder_id.split(":/")[0],
+            "id": root_folder_id,
+            "encrypt_uploads": True,  # TODO: somehow include this into settings
+        }
+
     async def get_item_info(self, item_id: str) -> storage.ItemResult:
         if item_id and ":/" in item_id:
             # All item_ids should contain a '/'
@@ -96,20 +103,21 @@ async def list_child_items(
                 for folder in response["CommonPrefixes"]:
                     results.append(
                         storage.ItemResult(
-                            item_id=f'{item_id}{folder["Prefix"]}',
-                            item_name=folder["Prefix"],
+                            item_id=f'{bucket}:/{folder["Prefix"]}',
+                            item_name=folder["Prefix"].split("/")[-2] + "/",
                             item_type=storage.ItemType.FOLDER,
                         )
                     )
             if response.get("Contents") and (item_type is not storage.ItemType.FOLDER):
                 for file in response["Contents"]:
-                    results.append(
-                        storage.ItemResult(
-                            item_id=f'{item_id}{file["Key"]}',
-                            item_name=file["Key"],
-                            item_type=storage.ItemType.FILE,
+                    if not file["Key"].endswith("/"):
+                        results.append(
+                            storage.ItemResult(
+                                item_id=f'{bucket}:/{file["Key"]}',
+                                item_name=file["Key"],
+                                item_type=storage.ItemType.FILE,
+                            )
                         )
-                    )
             return storage.ItemSampleResult(
                 items=results,
                 total_count=len(results),

addon_imps/tests/storage/test_figshare.py

@@ -103,7 +103,7 @@ async def test_item_info_article(self):
         self.imp._fetch_project = MagicMock(spec_set=self.imp._fetch_project)
 
         self.assertEqual(
-            await self.imp.get_item_info("articles/123"), sentinel.item_result
+            await self.imp.get_item_info("article/123"), sentinel.item_result
         )
 
         self.imp._fetch_article.assert_called_once_with("123")
@@ -116,7 +116,7 @@ async def test_item_info_project(self):
             return_value=MagicMock(item_result=sentinel.item_result),
         )
         self.assertEqual(
-            await self.imp.get_item_info("projects/123"), sentinel.item_result
+            await self.imp.get_item_info("project/123"), sentinel.item_result
         )
 
         self.imp._fetch_project.assert_called_once_with("123")
@@ -142,28 +142,28 @@ async def test_list_child_items(self):
 
         cases = [
             _ListChildItemsArgs(
-                "projects/123",
+                "project/123",
                 "123",
                 ItemType.FOLDER,
                 "_fetch_project_articles",
                 expected_positive_result,
             ),
             _ListChildItemsArgs(
-                "projects/123",
+                "project/123",
                 "123",
                 None,
                 "_fetch_project_articles",
                 expected_positive_result,
             ),
             _ListChildItemsArgs(
-                "articles/123",
+                "article/123",
                 "123",
                 ItemType.FOLDER,
                 "_fetch_article_files",
                 expected_negative_result,
             ),
             _ListChildItemsArgs(
-                "projects/123",
+                "project/123",
                 "123",
                 ItemType.FILE,
                 "_fetch_project_articles",
@@ -181,7 +181,7 @@ async def test_list_child_items(self):
             ],
             *[
                 _ListChildItemsArgs(
-                    "articles/345",
+                    "article/345",
                     "345",
                     item_type,
                     "_fetch_article_files",

addon_service/common/waterbutler_compat.py

@@ -1,12 +1,15 @@
+from asgiref.sync import async_to_sync
 from rest_framework_json_api import serializers
 
+from addon_service.addon_imp.instantiation import get_storage_addon_instance__blocking
+from addon_service.configured_addon.storage.models import ConfiguredStorageAddon
 from addon_toolkit import (
     credentials,
     json_arguments,
 )
 
 
-class WaterButlerCredentialsSerializer(serializers.Serializer):
+class WaterButlerConfigSerializer(serializers.Serializer):
     """Serialize ConfiguredStorageAddon information required by WaterButler.
 
     The returned data should share a shape with the existing `serialize_waterbutler_credentials`
@@ -17,6 +20,7 @@ class JSONAPIMeta:
         resource_name = "waterbutler-config"
 
     credentials = serializers.SerializerMethodField("_credentials_for_waterbutler")
+    config = serializers.SerializerMethodField("_config_for_waterbutler")
 
     def _credentials_for_waterbutler(self, configured_storage_addon):
         _creds_data = configured_storage_addon.credentials
@@ -31,3 +35,15 @@ def _credentials_for_waterbutler(self, configured_storage_addon):
                 return json_arguments.json_for_dataclass(_creds_data)
             case _:
                 raise ValueError(f"unknown credentials type: {_creds_data}")
+
+    def _config_for_waterbutler(self, configured_storage_addon: ConfiguredStorageAddon):
+        imp = get_storage_addon_instance__blocking(
+            configured_storage_addon.imp_cls,
+            configured_storage_addon.base_account,
+            configured_storage_addon.config,
+        )
+        imp_config = async_to_sync(imp.build_wb_config)(
+            configured_storage_addon.root_folder,
+            configured_storage_addon.external_service.wb_key,
+        )
+        return imp_config

addon_service/configured_addon/storage/views.py

@@ -4,7 +4,7 @@
 from rest_framework.response import Response
 
 from addon_service.common.credentials_formats import CredentialsFormats
-from addon_service.common.waterbutler_compat import WaterButlerCredentialsSerializer
+from addon_service.common.waterbutler_compat import WaterButlerConfigSerializer
 from addon_service.configured_addon.views import ConfiguredAddonViewSet
 
 from .models import ConfiguredStorageAddon
@@ -26,4 +26,4 @@ def get_wb_credentials(self, request, pk=None):
         if addon.external_service.credentials_format is CredentialsFormats.OAUTH2:
             addon.base_account.refresh_oauth_access_token__blocking()
         self.resource_name = "waterbutler-credentials"  # for the jsonapi resource type
-        return Response(WaterButlerCredentialsSerializer(addon).data)
+        return Response(WaterButlerConfigSerializer(addon).data)

addon_service/oauth2/models.py

@@ -116,7 +116,8 @@ def update_with_fresh_token(
     ) -> tuple[AuthorizedAccount]:
         # update this record's fields
         self.state_nonce = None  # one-time-use, now used
-        self.refresh_token = fresh_token_result.refresh_token
+        if fresh_token_result.refresh_token:
+            self.refresh_token = fresh_token_result.refresh_token
         if fresh_token_result.expires_in is None:
             self.access_token_expiration = None
         else:

addon_toolkit/interfaces/storage.py

@@ -152,6 +152,9 @@ class StorageAddonImp(AddonImp):
 
     config: StorageConfig
 
+    async def build_wb_config(self, root_folder_id: str, service_name: str) -> dict:
+        return {}
+
 
 @dataclasses.dataclass
 class StorageAddonHttpRequestorImp(StorageAddonImp):