Fix failure during gsctl install_deps and update the doc

coordinator/gscoordinator/coordinator.py

@@ -30,7 +30,7 @@
 from graphscope.config import Config
 from graphscope.proto import coordinator_service_pb2_grpc
 
-from gscoordinator.servicer import *
+from gscoordinator.servicer import init_graphscope_one_service_servicer
 from gscoordinator.utils import GS_GRPC_MAX_MESSAGE_LENGTH
 
 logger = logging.getLogger("graphscope")

coordinator/gscoordinator/servicer/graphscope_one/service.py

@@ -20,7 +20,6 @@
 
 import atexit
 import base64
-import functools
 import json
 import logging
 import os
@@ -63,6 +62,7 @@
 from gscoordinator.object_manager import ObjectManager
 from gscoordinator.op_executor import OperationExecutor
 from gscoordinator.operator_launcher import OperatorLauncher
+from gscoordinator.utils import catch_unknown_errors
 from gscoordinator.utils import check_server_ready
 from gscoordinator.utils import create_single_op_dag
 from gscoordinator.version import __version__
@@ -72,36 +72,6 @@
 logger = logging.getLogger("graphscope")
 
 
-def catch_unknown_errors(response_on_error=None, using_yield=False):
-    """A catcher that catches all (unknown) exceptions in gRPC handlers to ensure
-    the client not think the coordinator services is crashed.
-    """
-
-    def catch_exceptions(handler):
-        @functools.wraps(handler)
-        def handler_execution(self, request, context):
-            try:
-                if using_yield:
-                    for result in handler(self, request, context):
-                        yield result
-                else:
-                    yield handler(self, request, context)
-            except Exception as exc:
-                error_message = repr(exc)
-                error_traceback = traceback.format_exc()
-                context.set_code(grpc.StatusCode.ABORTED)
-                context.set_details(
-                    'Error occurs in handler: "%s", with traceback: ' % error_message
-                    + error_traceback
-                )
-                if response_on_error is not None:
-                    yield response_on_error
-
-        return handler_execution
-
-    return catch_exceptions
-
-
 class GraphScopeOneServiceServicer(
     coordinator_service_pb2_grpc.CoordinatorServiceServicer
 ):

coordinator/gscoordinator/utils.py

@@ -19,6 +19,7 @@
 
 import copy
 import datetime
+import functools
 import glob
 import hashlib
 import inspect
@@ -30,6 +31,7 @@
 import subprocess
 import sys
 import time
+import traceback
 import uuid
 import zipfile
 from concurrent.futures import ThreadPoolExecutor
@@ -38,6 +40,7 @@
 from string import Template
 from typing import List
 
+import grpc
 import yaml
 from google.protobuf.any_pb2 import Any
 from graphscope.framework import utils
@@ -182,6 +185,36 @@
 INTERACTIVE_ENGINE_THREADS_PER_WORKER = 2
 
 
+def catch_unknown_errors(response_on_error=None, using_yield=False):
+    """A catcher that catches all (unknown) exceptions in gRPC handlers to ensure
+    the client not think the coordinator services is crashed.
+    """
+
+    def catch_exceptions(handler):
+        @functools.wraps(handler)
+        def handler_execution(self, request, context):
+            try:
+                if using_yield:
+                    for result in handler(self, request, context):
+                        yield result
+                else:
+                    yield handler(self, request, context)
+            except Exception as exc:
+                error_message = repr(exc)
+                error_traceback = traceback.format_exc()
+                context.set_code(grpc.StatusCode.ABORTED)
+                context.set_details(
+                    'Error occurs in handler: "%s", with traceback: ' % error_message
+                    + error_traceback
+                )
+                if response_on_error is not None:
+                    yield response_on_error
+
+        return handler_execution
+
+    return catch_exceptions
+
+
 def get_timestamp() -> float:
     return datetime.datetime.timestamp(datetime.datetime.now())
 

docs/utilities/gs.md

@@ -23,11 +23,28 @@ This will install `gsctl` in an editable mode, which means that any changes you
 
 ## Commands
 
-With `gsctl`, you can do the following things. Always remember to
- use `--help` on a command to get more information.
+With `gsctl`, you can do the following things. Always remember to use `--help` on a command to get more information.
+
+The `gsctl` command-line utility supports two modes of operation: utility scripts and client/server mode. You can switch between these modes using the
+`gsctl connect` and `gsctl close` commands.
+
+### Utility Scripts
+
+Default, the `gsctl` provide helper functions and utilities that can be run using gsctl alone.
+`gsctl` acts as the command-line entrypoint for GraphScope. Some examples of utility scripts are:
 
 - `gsctl install-deps`, install dependencies for building GraphScope.
 - `gsctl make`, build GraphScope executable binaries and artifacts.
 - `gsctl make-image`, build GraphScope docker images.
 - `gsctl test`, trigger test suites.
+- `gsctl connect`, connect to the launched coordinator by ~/.gs/config.
+- `gsctl close`, Close the connection from the coordinator.
+
+
+### Client/Server Mode
+
+To switch to the client/server mode, use the `gsctl connect` command. This command connects gsctl to a launched coordinator using the configuration file located at ~/.gsconfig.
+Once connected, you can use `gsctl` to communicate with the coordinator and send commands that will be executed on the coordinator side.
 
+To disconnect from the coordinator and switch back to the utility scripts mode, you can use the `gsctl close` command. This command closes the connection from the coordinator
+and allows you to use `gsctl` as a standalone utility again.

python/graphscope/gsctl/commands/common_command.py

@@ -37,7 +37,7 @@ def cli():
     help="Coordinator endpoint which gsctl connect to, e.g. http://127.0.0.1:9527",
 )
 def connect(coordinator_endpoint):
-    """Connect to the launched coordinator by ~/.gs/config. If '--coordinator-endpoint' is specified,
+    """Connect to the launched coordinator by ~/.graphscope/config. If '--coordinator-endpoint' is specified,
     use it as the current context and override the config file.
     """
     if coordinator_endpoint is not None:
@@ -53,7 +53,7 @@ def connect(coordinator_endpoint):
         config = load_gs_config()
         config.set_and_write(context)
 
-    click.secho(f"Coordinator service connected.", fg="green")
+    click.secho("Coordinator service connected.", fg="green")
 
 
 @click.command()

python/graphscope/gsctl/commands/dev_command.py

@@ -57,8 +57,8 @@ def cli():
     required=True,
 )
 @click.option(
-    "--repo-home",
-    envvar="REPO_HOME",
+    "--graphscope-repo",
+    envvar="GRAPHSCOPE_REPO",
     type=click.Path(),
     default=os.path.abspath("."),
     show_default=True,
@@ -112,7 +112,7 @@ def cli():
 )
 def install_deps(
     type,
-    repo_home,
+    graphscope_repo,
     cn,
     install_prefix,
     from_local,
@@ -145,7 +145,7 @@ def install_deps(
         cmd.append("--no-v6d")
     if cn:
         cmd.append("--cn")
-    run_shell_cmd(cmd, repo_home)
+    run_shell_cmd(cmd, graphscope_repo)
 
 
 @click.command()
@@ -170,8 +170,8 @@ def install_deps(
     required=False,
 )
 @click.option(
-    "--repo-home",
-    envvar="REPO_HOME",
+    "--graphscope-repo",
+    envvar="GRAPHSCOPE_REPO",
     type=click.Path(),
     default=os.path.abspath("."),
     show_default=True,
@@ -189,7 +189,7 @@ def install_deps(
     default="default",
     help="Make gie with specified storage type.",
 )
-def make(component, repo_home, install_prefix, storage_type):
+def make(component, graphscope_repo, install_prefix, storage_type):
     """Build executive binaries of COMPONENT. If not given a specific component, build all.
     \f
     TODO: maybe without make?
@@ -199,7 +199,7 @@ def make(component, repo_home, install_prefix, storage_type):
         fg="yellow",
     )
     click.secho(
-        f"Begin the make command, to build components [{component}] of GraphScope, with repo = {repo_home}",
+        f"Begin the make command, to build components [{component}] of GraphScope, with repo = {graphscope_repo}",
         fg="green",
     )
     if component is None:
@@ -215,7 +215,7 @@ def make(component, repo_home, install_prefix, storage_type):
         "-s",
         storage_type,
     ]
-    run_shell_cmd(cmd, repo_home)
+    run_shell_cmd(cmd, graphscope_repo)
 
 
 @click.command()
@@ -241,8 +241,8 @@ def make(component, repo_home, install_prefix, storage_type):
     required=False,
 )
 @click.option(
-    "--repo-home",
-    envvar="REPO_HOME",
+    "--graphscope-repo",
+    envvar="GRAPHSCOPE_REPO",
     type=click.Path(),
     default=os.path.abspath("."),
     show_default=True,
@@ -260,7 +260,7 @@ def make(component, repo_home, install_prefix, storage_type):
     show_default=True,
     help="registry name",
 )
-def make_image(component, repo_home, registry, tag):
+def make_image(component, graphscope_repo, registry, tag):
     """Make docker images from source code for deployment.
     \f
     TODO: fulfill this.
@@ -269,7 +269,7 @@ def make_image(component, repo_home, registry, tag):
         component = "all"
 
     cmd = ["bash", make_image_script, "-c", component, "-r", registry, "-t", tag]
-    run_shell_cmd(cmd, repo_home)
+    run_shell_cmd(cmd, graphscope_repo)
 
 
 @click.command()
@@ -290,8 +290,8 @@ def make_image(component, repo_home, registry, tag):
     required=False,
 )
 @click.option(
-    "--repo-home",
-    envvar="REPO_HOME",
+    "--graphscope-repo",
+    envvar="GRAPHSCOPE_REPO",
     type=click.Path(),
     default=os.path.abspath("."),
     show_default=True,
@@ -329,12 +329,12 @@ def make_image(component, repo_home, registry, tag):
     default=False,
     help="Run nx tests",
 )
-def test(type, repo_home, testdata, local, storage_type, k8s, nx):
+def test(type, graphscope_repo, testdata, local, storage_type, k8s, nx):
     """Trigger tests on built artifacts.
 
     \f
     TODO: fulfill this."""
-    click.secho(f"repo_home = {repo_home}", fg="green")
+    click.secho(f"graphscope_repo = {graphscope_repo}", fg="green")
     click.echo("test")
     if type is None:
         type = ""
@@ -354,7 +354,7 @@ def test(type, repo_home, testdata, local, storage_type, k8s, nx):
         "-n",
         str(nx),
     ]
-    run_shell_cmd(cmd, repo_home)
+    run_shell_cmd(cmd, graphscope_repo)
 
 
 cli.add_command(install_deps)

python/graphscope/gsctl/commands/utils.py

@@ -28,10 +28,13 @@ def get_command_collection(context: Context):
         # treat gsctl as an utility script, providing hepler functions or utilities. e.g.
         # initialize and manage cluster, install the dependencies required to build graphscope locally
         commands = click.CommandCollection(sources=[common_cli, dev_cli])
+
     elif context.solution == "interactive":
         commands = click.CommandCollection(sources=[common_cli])
+
     else:
         raise RuntimeError(
             f"Failed to get command collection with context {context.name}"
         )
+
     return commands

python/graphscope/gsctl/config.py

@@ -25,7 +25,7 @@
 import yaml
 
 GS_CONFIG_DEFAULT_LOCATION = os.environ.get(
-    "GSCONFIG", os.path.expanduser("~/.gs/config")
+    "GSCONFIG", os.path.expanduser("~/.graphscope/config")
 )
 
 
@@ -34,7 +34,7 @@ def __init__(self, solution, coordinator_endpoint, name=None):
         self.supported_solutions = ["interactive"]
         if solution not in self.supported_solutions:
             raise RuntimeError(
-                "The solution {0} in context {01 is not supported yet.".format(
+                "The solution {0} in context {1} is not supported yet.".format(
                     solution, name
                 )
             )

python/graphscope/gsctl/gsctl.py

@@ -16,13 +16,31 @@
 # limitations under the License.
 #
 
+import os
+import sys
+
 import click
 
-from graphscope.gsctl.commands import get_command_collection
-from graphscope.gsctl.config import get_current_context
+try:
+    import graphscope
+except ModuleNotFoundError:
+    # if graphscope is not installed, only basic functions or utilities
+    # can be used, e.g. install dependencies
+    graphscope = None
 
 
 def cli():
+    if graphscope is None:
+        sys.path.insert(
+            0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "commands")
+        )
+        from dev_command import cli as dev_cli
+
+        dev_cli()
+
+    from graphscope.gsctl.commands import get_command_collection
+    from graphscope.gsctl.config import get_current_context
+
     context = get_current_context()
     # get the specified commands under the FLEX architecture
     commands = get_command_collection(context)