Merge pull request #170 from opaduchak/feature/account-migrations

[ENG-5701][ENG-5702] Account migrations

addon_imps/storage/dataverse.py

@@ -32,7 +32,7 @@ async def build_wb_config(
         self,
     ) -> dict:
         match = DATASET_REGEX.match(self.config.connected_root_id)
-        async with self.network.GET(f"datasets/{match['id']}") as response:
+        async with self.network.GET(f"api/datasets/{match['id']}") as response:
             content = await response.json_content()
             parsed = parse_dataset(content)
             return {
@@ -44,7 +44,7 @@ async def build_wb_config(
 
     async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResult:
         async with self.network.GET(
-            "mydata/retrieve",
+            "api/mydata/retrieve",
             query=[
                 ["page", page_cursor],
                 *[("role_ids", role) for role in range(1, 9)],
@@ -104,7 +104,9 @@ async def list_child_items(
             return ItemSampleResult(items=[], total_count=0)
 
     async def _fetch_dataverse_items(self, dataverse_id) -> list[ItemResult]:
-        async with self.network.GET(f"dataverses/{dataverse_id}/contents") as response:
+        async with self.network.GET(
+            f"api/dataverses/{dataverse_id}/contents"
+        ) as response:
             response_content = await response.json_content()
             return await asyncio.gather(
                 *[
@@ -122,19 +124,19 @@ async def get_dataverse_or_dataset_item(self, item: dict):
         raise ValueError(f"Invalid item type: {item['type']}")
 
     async def _fetch_dataverse(self, dataverse_id) -> ItemResult:
-        async with self.network.GET(f"dataverses/{dataverse_id}") as response:
+        async with self.network.GET(f"api/dataverses/{dataverse_id}") as response:
             return parse_dataverse(await response.json_content())
 
     async def _fetch_dataset(self, dataset_id: str) -> ItemResult:
-        async with self.network.GET(f"datasets/{dataset_id}") as response:
+        async with self.network.GET(f"api/datasets/{dataset_id}") as response:
             return parse_dataset(await response.json_content())
 
     async def _fetch_dataset_files(self, dataset_id) -> list[ItemResult]:
-        async with self.network.GET(f"datasets/{dataset_id}") as response:
+        async with self.network.GET(f"api/datasets/{dataset_id}") as response:
             return parse_dataset_files(await response.json_content())
 
     async def _fetch_file(self, dataverse_id) -> ItemResult:
-        async with self.network.GET(f"files/{dataverse_id}") as response:
+        async with self.network.GET(f"api/files/{dataverse_id}") as response:
             return parse_datafile(await response.json_content())
 
 
@@ -147,6 +149,7 @@ def parse_dataverse_as_subitem(data: dict):
         item_type=ItemType.FOLDER,
         item_name=data["title"],
         item_id=f'dataverse/{data["id"]}',
+        can_be_root=False,
     )
 
 

addon_imps/storage/dropbox.py

@@ -14,9 +14,20 @@ async def get_external_account_id(self, _: dict[str, str]) -> str:
         return ""
 
     async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResult:
-        return await self.list_child_items(item_id="", page_cursor=page_cursor)
+        return storage.ItemSampleResult(
+            items=[
+                storage.ItemResult(
+                    item_id="/",
+                    item_name="Root",
+                    item_type=ItemType.FOLDER,
+                )
+            ],
+            total_count=1,
+        )
 
     async def build_wb_config(self) -> dict:
+        if not self.config.connected_root_id or self.config.connected_root_id == "/":
+            return {"folder": self.config.connected_root_id}
         async with self.network.POST(
             "files/get_metadata",
             json={

addon_imps/storage/gitlab.py

@@ -29,7 +29,7 @@ class GitlabStorageImp(storage.StorageAddonHttpRequestorImp):
     """
 
     async def get_external_account_id(self, _: dict[str, str]) -> str:
-        async with self.network.GET("user/preferences") as response:
+        async with self.network.GET("api/v4/user/preferences") as response:
             resp_json = await response.json_content()
             return resp_json.get("user_id", "")
 
@@ -55,7 +55,7 @@ async def list_root_items(self, page_cursor: str = "") -> storage.ItemSampleResu
                 "sort": "asc",
             },
         )
-        async with self.network.GET("projects", query=query_params) as response:
+        async with self.network.GET("api/v4/projects", query=query_params) as response:
             resp = await response.json_content()
             return ItemSampleResult(
                 items=[Repository.from_json(item).item_result for item in resp],
@@ -93,12 +93,12 @@ async def get_item_info(self, item_id: str) -> storage.ItemResult:
         return (await self._get_repository(parsed_id.repo_id)).item_result
 
     async def _get_repository(self, repo_id):
-        async with self.network.GET(f"projects/{repo_id}") as response:
+        async with self.network.GET(f"api/v4/projects/{repo_id}") as response:
             content = await response.json_content()
             return Repository.from_json(content)
 
     async def get_file_or_folder(self, parsed_id: ItemId):
-        async with self.network.GET(f"projects/{parsed_id.repo_id}") as response:
+        async with self.network.GET(f"api/v4/projects/{parsed_id.repo_id}") as response:
             content = await response.json_content()
             ref = content.get("default_branch")
         if file_item := await self._get_file(parsed_id, ref):
@@ -113,7 +113,7 @@ async def get_file_or_folder(self, parsed_id: ItemId):
 
     async def _get_file(self, parsed_id, ref):
         async with self.network.GET(
-            f"projects/{parsed_id.repo_id}/repository/files/{quote_plus(parsed_id.file_path)}",
+            f"api/v4/projects/{parsed_id.repo_id}/repository/files/{quote_plus(parsed_id.file_path)}",
             query={"ref": ref},
         ) as response:
             content = await response.json_content()
@@ -142,7 +142,7 @@ async def list_child_items(
             },
         )
         async with self.network.GET(
-            f"projects/{parsed_id.repo_id}/repository/tree",
+            f"api/v4/projects/{parsed_id.repo_id}/repository/tree",
             query=query_params,
         ) as response:
             if response.http_status == HTTPStatus.NOT_FOUND:

addon_imps/tests/storage/test_dropbox.py

@@ -51,16 +51,13 @@ async def test_list_root_items(self):
 
         expected_result = ItemSampleResult(
             items=[
-                ItemResult(
-                    item_id="123", item_name="root folder", item_type=ItemType.FOLDER
-                )
+                ItemResult(item_id="/", item_name="Root", item_type=ItemType.FOLDER)
             ],
             total_count=1,
-            next_sample_cursor="test_cursor",
         )
 
         self.assertEqual(result, expected_result)
-        self._assert_post("files/list_folder", json={"path": "", "recursive": False})
+        self.network.POST.assert_not_called()
 
     async def test_get_item_info(self):
         cases = [("", "root folder"), ("file_id", "file")]

addon_imps/tests/storage/test_gitlab.py

@@ -28,7 +28,7 @@ def _patch_get(self, return_value: dict | list, status=200, headers=None):
 
     def _assert_get(self, url: str, query: dict = None):
         extra_params = {"query": query} if query else {}
-        self.network.GET.assert_called_once_with(url, **extra_params)
+        self.network.GET.assert_called_once_with(f"api/v4/{url}", **extra_params)
         self.network.GET.return_value.__aenter__.assert_awaited_once()
         self.network.GET.return_value.__aenter__.return_value.json_content.assert_awaited_once()
         self.network.GET.return_value.__aexit__.assert_awaited_once_with(

addon_service/authorized_account/models.py

@@ -316,7 +316,7 @@ async def refresh_oauth2_access_token(self) -> None:
             _oauth_client_config,
             _oauth_token_metadata,
         ) = await self._load_oauth2_client_config_and_token_metadata()
-        if sync_to_async(lambda: _oauth_token_metadata.access_token_only)():
+        if await sync_to_async(lambda: _oauth_token_metadata.access_token_only)():
             return
         _fresh_token_result = await oauth2_utils.get_refreshed_access_token(
             token_endpoint_url=_oauth_client_config.token_endpoint_url,

addon_service/management/commands/fill_garbage.py

@@ -3,7 +3,7 @@
 
 from addon_service import models as db
 from addon_service.common import known_imps
-from addon_service.credentials.enums import CredentialsFormats
+from addon_service.common.credentials_formats import CredentialsFormats
 from addon_toolkit import AddonCapabilities