feat: Start the service when switching graph context on Interactive (#…

…3910)  ## What do these changes do? - Start the service when switching graph context - Move `gsctl service <start/stop/status>` from GLOBAL to GRAPH context - Move `gsctl create/delete datasource/loaderjob` from GRAPH to GLOBAL context - Display GRAPH_NAME(GRAPH_ID) in `gsctl service status` and `gsctl use GRAPH/GLOBAL` command - Return metadata when uploading file - Add `Y/N` interaction when deleting graph/storedproc/job resource ## Related issue number  Fixes --------- Co-authored-by: Zhang Lei <[email protected]>
alibaba · Jun 14, 2024 · 626501f · 626501f
1 parent aa60805
commit 626501f
Show file tree

Hide file tree

Showing 17 changed files with 354 additions and 217 deletions.
diff --git a/Makefile b/Makefile
@@ -79,9 +79,7 @@ clean:
 
 gsctl:
 	cd $(CLIENT_DIR) && \
-	python3 setup_gsctl.py bdist_wheel && \
-	python3 -m pip install dist/gsctl*.whl --force-reinstall && \
-	rm -fr build
+	python3 ./setup_gsctl.py develop --user
 
 
 client: learning

diff --git a/coordinator/gscoordinator/flex/core/client_wrapper.py b/coordinator/gscoordinator/flex/core/client_wrapper.py
@@ -37,6 +37,7 @@
 from gscoordinator.flex.core.insight import init_groot_client
 from gscoordinator.flex.core.interactive import init_hqps_client
 from gscoordinator.flex.core.utils import encode_datetime
+from gscoordinator.flex.core.utils import parse_file_metadata
 from gscoordinator.flex.models import CreateDataloadingJobResponse
 from gscoordinator.flex.models import CreateEdgeType
 from gscoordinator.flex.models import CreateGraphRequest
@@ -289,7 +290,8 @@ def upload_file(self, filestorage) -> str:
         if CLUSTER_TYPE == "HOSTS":
             filepath = os.path.join(DATASET_WORKSPACE, filestorage.filename)
             filestorage.save(filepath)
-            return UploadFileResponse.from_dict({"file_path": filepath})
+            metadata = parse_file_metadata(filepath)
+            return UploadFileResponse.from_dict({"file_path": filepath, "metadata": metadata})
 
     def bind_datasource_in_batch(
         self, graph_id: str, schema_mapping: SchemaMapping
@@ -301,9 +303,10 @@ def bind_datasource_in_batch(
             schema_mapping_dict["vertex_mappings"],
             schema_mapping_dict["edge_mappings"],
         ):
-            for column_mapping in mapping["column_mappings"]:
-                if "_property" in column_mapping:
-                    column_mapping["property"] = column_mapping.pop("_property")
+            if "column_mappings" in mapping and mapping["column_mappings"] is not None:
+                for column_mapping in mapping["column_mappings"]:
+                    if "_property" in column_mapping:
+                        column_mapping["property"] = column_mapping.pop("_property")
             if (
                 "source_vertex_mappings" in mapping
                 and "destination_vertex_mappings" in mapping

diff --git a/coordinator/gscoordinator/flex/core/utils.py b/coordinator/gscoordinator/flex/core/utils.py
@@ -19,7 +19,9 @@
 import datetime
 import functools
 import logging
+import os
 import random
+import re
 import socket
 import string
 from typing import Union
@@ -83,6 +85,28 @@ def get_internal_ip() -> str:
     return internal_ip
 
 
+def parse_file_metadata(location: str) -> dict:
+    """
+    Args:
+        location: optional values:
+            odps://path/to/file, hdfs://path/to/file, file:///path/to/file
+            /home/graphscope/path/to/file
+    """
+    metadata = {"datasource": "file"}
+    path = location
+    pattern = r"^(odps|hdfs|file|oss|s3)?://([\w/.-]+)$"
+    match = re.match(pattern, location)
+    if match:
+        datasource = match.group(1)
+        metadata["datasource"] = datasource
+        if datasource == "file":
+            path = match.group(2)
+    if metadata["datasource"] == "file":
+        _, file_extension = os.path.splitext(path)
+        metadata["file_type"] = file_extension[1:]
+    return metadata
+
+
 def get_public_ip() -> Union[str, None]:
     try:
         response = requests.get("https://api.ipify.org?format=json")

diff --git a/coordinator/gscoordinator/flex/models/upload_file_response.py b/coordinator/gscoordinator/flex/models/upload_file_response.py
@@ -12,21 +12,26 @@ class UploadFileResponse(Model):
     Do not edit the class manually.
     """
 
-    def __init__(self, file_path=None):  # noqa: E501
+    def __init__(self, file_path=None, metadata=None):  # noqa: E501
         """UploadFileResponse - a model defined in OpenAPI
 
         :param file_path: The file_path of this UploadFileResponse.  # noqa: E501
         :type file_path: str
+        :param metadata: The metadata of this UploadFileResponse.  # noqa: E501
+        :type metadata: Dict[str, object]
         """
         self.openapi_types = {
-            'file_path': str
+            'file_path': str,
+            'metadata': Dict[str, object]
         }
 
         self.attribute_map = {
-            'file_path': 'file_path'
+            'file_path': 'file_path',
+            'metadata': 'metadata'
         }
 
         self._file_path = file_path
+        self._metadata = metadata
 
     @classmethod
     def from_dict(cls, dikt) -> 'UploadFileResponse':
@@ -61,3 +66,26 @@ def file_path(self, file_path: str):
             raise ValueError("Invalid value for `file_path`, must not be `None`")  # noqa: E501
 
         self._file_path = file_path
+
+    @property
+    def metadata(self) -> Dict[str, object]:
+        """Gets the metadata of this UploadFileResponse.
+
+
+        :return: The metadata of this UploadFileResponse.
+        :rtype: Dict[str, object]
+        """
+        return self._metadata
+
+    @metadata.setter
+    def metadata(self, metadata: Dict[str, object]):
+        """Sets the metadata of this UploadFileResponse.
+
+
+        :param metadata: The metadata of this UploadFileResponse.
+        :type metadata: Dict[str, object]
+        """
+        if metadata is None:
+            raise ValueError("Invalid value for `metadata`, must not be `None`")  # noqa: E501
+
+        self._metadata = metadata
diff --git a/coordinator/gscoordinator/flex/openapi/openapi.yaml b/coordinator/gscoordinator/flex/openapi/openapi.yaml
@@ -473,6 +473,11 @@ paths:
         "200":
           content:
             application/json:
+              example:
+                file_path: /home/graphscope/path/to/file.csv
+                metadata:
+                  datasource: file
+                  file_type: csv
               schema:
                 $ref: '#/components/schemas/UploadFileResponse'
           description: successful operation
@@ -2008,12 +2013,19 @@ components:
     UploadFileResponse:
       example:
         file_path: file_path
+        metadata:
+          key: ""
       properties:
         file_path:
           title: file_path
           type: string
+        metadata:
+          additionalProperties: true
+          title: metadata
+          type: object
       required:
       - file_path
+      - metadata
       title: UploadFileResponse
     LongText:
       properties:

diff --git a/docs/flex/coordinator.md b/docs/flex/coordinator.md
@@ -1,4 +1,4 @@
-# GraphScope Coordinator
+# Coordinator
 
 The GraphScope Coordinator serves as a centralized entry point for users, providing a RESTful API that follows the Swagger specification. It supports multiple language SDKs, including Python, and offers a unified interface. The main purpose of the Coordinator is to abstract and standardize the underlying engines and storage systems, shielding users from their complexities. This allows users to interact with the GraphScope platform through a simplified and consistent set of APIs, making it easier for users to understand and utilize the functionalities provided by GraphScope.
 

diff --git a/docs/utilities/gs.md b/docs/utilities/gs.md
@@ -1,23 +1,22 @@
 # Command-line Utility `gsctl`
 
-`gsctl` is a command-line utility for GraphScope. It is shipped with `graphscope-client` and provides a set of functionalities to make it easy to use GraphScope. These functionalities include building and testing binaries, managing sessions and resources, and more.
+`gsctl` is a command-line utility for GraphScope. It provides a set of functionalities to make it easy to use GraphScope. These functionalities include building and testing binaries, managing sessions and resources, and more.
 
 ## Install/Update `gsctl`
 
-Since it is shipped with python package `graphscope-client`, the `gsctl` command will be available in your terminal after installing GraphScope:
 ```bash
-$ pip3 install graphscope-client
+$ pip3 install gsctl
 ```
 
 In some cases, such as development on `gsctl`, you may want to build it from source.
 To do this, navigate to the directory where the source code is located and run the following command:
 
 ```bash
-$ cd REPO_HOME/python
+$ cd REPO_HOME
 # If you want to develop gsctl,
 # please note the entry point is located on:
 # /python/graphscope/gsctl/gsctl.py
-$ pip3 install --editable .
+$ make gsctl
 ```
 This will install `gsctl` in an editable mode, which means that any changes you make to the source code will be reflected in the installed version of `gsctl`.
 

diff --git a/flex/openapi/openapi_coordinator.yaml b/flex/openapi/openapi_coordinator.yaml
@@ -169,9 +169,13 @@ components:
     UploadFileResponse:
       required:
         - file_path
+        - metadata
       properties:
         file_path:
           type: string
+        metadata:
+          type: object
+          additionalProperties: true
 
     LongText:
       required:
@@ -2297,5 +2301,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/UploadFileResponse'
+              example:
+                file_path: /home/graphscope/path/to/file.csv
+                metadata:
+                  datasource: file
+                  file_type: csv
         500:
           $ref: "#/components/responses/500"
diff --git a/python/graphscope/flex/rest/models/upload_file_response.py b/python/graphscope/flex/rest/models/upload_file_response.py
@@ -28,7 +28,8 @@ class UploadFileResponse(BaseModel):
     UploadFileResponse
     """ # noqa: E501
     file_path: StrictStr
-    __properties: ClassVar[List[str]] = ["file_path"]
+    metadata: Dict[str, Any]
+    __properties: ClassVar[List[str]] = ["file_path", "metadata"]
 
     model_config = {
         "populate_by_name": True,
@@ -81,7 +82,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
             return cls.model_validate(obj)
 
         _obj = cls.model_validate({
-            "file_path": obj.get("file_path")
+            "file_path": obj.get("file_path"),
+            "metadata": obj.get("metadata")
         })
         return _obj
 

diff --git a/python/graphscope/gsctl/commands/__init__.py b/python/graphscope/gsctl/commands/__init__.py
@@ -108,7 +108,11 @@ def get_command_collection(context: Context):
             commands = click.CommandCollection(sources=[common, interactive])
         else:
             if len(sys.argv) < 2 or sys.argv[1] != "use":
-                info(f"Using GRAPH {context.context}.", fg="green", bold=True)
+                info(
+                    f"Using GRAPH {context.graph_name}(id={context.context}).",
+                    fg="green",
+                    bold=True,
+                )
                 info("Run `gsctl use GLOBAL` to switch back to GLOBAL context.\n")
             commands = click.CommandCollection(sources=[common, interactive_graph])
     elif is_insight_mode(context.flex):