Add GremlinServiceAvailable alert rule

flex/coordinator/gs_flex_coordinator/core/alert/builtin_rules.py

@@ -21,8 +21,11 @@
 import socket
 
 import psutil
+from gremlin_python.driver.client import Client
+from gs_flex_coordinator.core import client_wrapper
 from gs_flex_coordinator.core.alert.alert_rule import AlertRule
-from gs_flex_coordinator.core.alert.message_collector import AlertMessageCollector
+from gs_flex_coordinator.core.alert.message_collector import \
+    AlertMessageCollector
 from gs_flex_coordinator.core.config import CLUSTER_TYPE, SOLUTION
 
 logger = logging.getLogger("graphscope")
@@ -74,6 +77,52 @@ def run_alert(self):
             logger.warn("Failed to get disk usage: %s", str(e))
 
 
+class GremlinServiceAvailableAlert(AlertRule):
+    def __init__(
+        self,
+        name,
+        severity,
+        metric_type,
+        conditions_description,
+        frequency,
+        message_collector,
+        enable=True,
+    ):
+        super().__init__(
+            name,
+            severity,
+            metric_type,
+            conditions_description,
+            frequency,
+            message_collector,
+            enable,
+        )
+
+    def run_alert(self):
+        """This function needs to handle exception by itself"""
+        try:
+            graph = client_wrapper.get_current_graph()
+            gremlin_interface = graph.gremlin_interface
+            client = Client(
+                gremlin_interface["gremlin_endpoint"],
+                "g",
+                username=gremlin_interface["username"],
+                password=gremlin_interface["password"],
+            )
+            client.submit("g.with('evaluationTimeout', 5000).V().limit(1)").all.result()
+        except Exception as e:
+            message = "Gremlin service unavailable: {0}".format(str(e))
+            # unable to distinguish whether frontend or excutor is unavailable,
+            # thus, we set the target "-"
+            alert_message = self.generate_alert_message("-", message)
+            self.alert(alert_message)
+        finally:
+            try:
+                client.close()
+            except:  # noqa: E722
+                pass
+
+
 def init_builtin_alert_rules(message_collector: AlertMessageCollector):
     alert_rules = {}
     # HighDiskUtilization
@@ -82,9 +131,19 @@ def init_builtin_alert_rules(message_collector: AlertMessageCollector):
         severity="warning",
         metric_type="node",
         conditions_description="disk_utilization>80",
-        frequency=1,
+        frequency=180,
         message_collector=message_collector,
         threshold=80,
         enable=True,
     )
+    if SOLUTION == "GRAPHSCOPE_INSIGHT":
+        alert_rules["GremlinServiceAvailable"] = GremlinServiceAvailableAlert(
+            name="GremlinServiceAvailable",
+            severity="emergency",
+            metric_type="service",
+            conditions_description="g.V().limit(1) failed",
+            frequency=5,
+            message_collector=message_collector,
+            enable=True,
+        )
     return alert_rules

flex/coordinator/gs_flex_coordinator/core/client_wrapper.py

@@ -26,7 +26,6 @@
 from typing import List, Union
 
 import psutil
-
 from gs_flex_coordinator.core.config import (CLUSTER_TYPE, CREATION_TIME,
                                              DATASET_WORKSPACE, INSTANCE_NAME,
                                              SOLUTION, WORKSPACE)
@@ -37,11 +36,11 @@
                                             encode_datetime, get_current_time)
 from gs_flex_coordinator.models import (DataSource, DeploymentInfo,
                                         EdgeDataSource, EdgeType, Graph,
-                                        GrootGraph, GrootSchema, JobStatus,
-                                        ModelSchema, NodeStatus, Procedure,
-                                        SchemaMapping, ServiceStatus,
-                                        StartServiceRequest, VertexDataSource,
-                                        VertexType, GrootDataloadingJobConfig)
+                                        GrootDataloadingJobConfig, GrootGraph,
+                                        GrootSchema, JobStatus, ModelSchema,
+                                        NodeStatus, Procedure, SchemaMapping,
+                                        ServiceStatus, StartServiceRequest,
+                                        VertexDataSource, VertexType)
 from gs_flex_coordinator.version import __version__
 
 logger = logging.getLogger("graphscope")
@@ -126,6 +125,9 @@ def get_groot_schema(self, graph_name: str) -> GrootSchema:
     def import_groot_schema(self, graph_name: str, schema: GrootSchema) -> str:
         return self._client.import_groot_schema(graph_name, schema.to_dict())
 
+    def get_current_graph(self) -> GrootGraph:
+        return self._client.get_current_graph()
+
     def create_graph(self, graph: Graph) -> str:
         # there are some tricks here, since schema is a keyword of openapi
         # specification, so it will be converted into the _schema field.
@@ -282,7 +284,9 @@ def upload_file(self, filestorage) -> str:
     def create_groot_dataloading_job(
         self, graph_name: str, job_config: GrootDataloadingJobConfig
     ) -> str:
-        job_id = self._client.create_groot_dataloading_job(graph_name, job_config.to_dict())
+        job_id = self._client.create_groot_dataloading_job(
+            graph_name, job_config.to_dict()
+        )
         return job_id
 
     def list_groot_graph(self) -> List[GrootGraph]:

flex/coordinator/gs_flex_coordinator/core/insight/graph.py

@@ -26,9 +26,6 @@
 from graphscope.deploy.kubernetes.utils import (get_service_endpoints,
                                                 resolve_api_client)
 from gremlin_python.driver.client import Client
-from kubernetes import client as kube_client
-from kubernetes import config as kube_config
-
 from gs_flex_coordinator.core.config import (CLUSTER_TYPE, CREATION_TIME,
                                              ENABLE_DNS, GROOT_GREMLIN_PORT,
                                              GROOT_GRPC_PORT, GROOT_PASSWORD,
@@ -39,6 +36,8 @@
                                             encode_datetime, get_internal_ip,
                                             get_public_ip)
 from gs_flex_coordinator.version import __version__
+from kubernetes import client as kube_client
+from kubernetes import config as kube_config
 
 logger = logging.getLogger("graphscope")
 
@@ -283,12 +282,6 @@ def get_groot_graph_from_local():
             client.submit(
                 "g.with('evaluationTimeout', 5000).V().limit(1)"
             ).all().result()
-            print(
-                "DEBUG: ",
-                client.submit("g.with('evaluationTimeout', 5000).V().valueMap().limit(10)")
-                .all()
-                .result(),
-            )
         except Exception as e:
             pass
         else:

flex/coordinator/gs_flex_coordinator/core/insight/groot.py

@@ -106,6 +106,9 @@ def get_edge_full_label(
     ) -> str:
         return f"{source_vertex_type}_{type_name}_{destination_vertex_type}"
 
+    def get_current_graph(self):
+        return self._graph
+
     def list_groot_graph(self) -> list:
         rlts = [self._graph.to_dict()]
         return rlts