From 8bb571067e53ae60b494ee3c930870aa98034b0d Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 17 Jul 2024 11:29:40 +0800 Subject: [PATCH 01/10] make use of cache when building interactive image Committed-by: xiaolei.zl from Dev container --- flex/interactive/docker/Makefile | 2 +- flex/interactive/docker/interactive-runtime.Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flex/interactive/docker/Makefile b/flex/interactive/docker/Makefile index 4eba0d8a62e4..b55f4cd832b0 100644 --- a/flex/interactive/docker/Makefile +++ b/flex/interactive/docker/Makefile @@ -31,7 +31,7 @@ interactive-runtime: --target final_image \ --build-arg ARCH=$(ARCH) \ --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ - --no-cache -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-${ARCH} . + -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-${ARCH} . hqps-server-base: docker build \ diff --git a/flex/interactive/docker/interactive-runtime.Dockerfile b/flex/interactive/docker/interactive-runtime.Dockerfile index d8529edfa85e..516edd8f57bc 100644 --- a/flex/interactive/docker/interactive-runtime.Dockerfile +++ b/flex/interactive/docker/interactive-runtime.Dockerfile @@ -4,8 +4,6 @@ FROM registry.cn-hongkong.aliyuncs.com/graphscope/interactive-base:v0.0.4 AS bui ARG ARCH ARG ENABLE_COORDINATOR="false" -COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope - # change bash as default SHELL ["/bin/bash", "-c"] @@ -30,6 +28,8 @@ cmake . -DCMAKE_INSTALL_PREFIX=/opt/flex -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_ -DWITH_ABSEIL=OFF -DWITH_PROMETHEUS=OFF \ -DBUILD_TESTING=OFF -DWITH_EXAMPLES=OFF && make -j && make install && rm -rf /tmp/opentelemetry-cpp +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope + # install flex RUN . ${HOME}/.cargo/env && cd ${HOME}/GraphScope/flex && \ git submodule update --init && mkdir build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=/opt/flex -DBUILD_DOC=OFF -DBUILD_TEST=OFF && make -j && make install && \ From 4cd66276eafbde18ee95a758be745e5e3799e1dc Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Thu, 1 Aug 2024 20:07:13 +0800 Subject: [PATCH 02/10] move image building to k8s/ Committed-by: xiaolei.zl from Dev container --- flex/interactive/docker/Makefile | 39 --------------- flex/interactive/docker/README.md | 35 -------------- .../docker/interactive-base.Dockerfile | 48 ------------------- k8s/Makefile | 15 +++++- .../dockerfiles/flex-interactive.Dockerfile | 12 +++-- .../dockerfiles/interactive-entrypoint.sh | 0 python/graphscope/gsctl/commands/dev.py | 4 +- .../gsctl/scripts/make_image_command.sh | 6 +++ 8 files changed, 29 insertions(+), 130 deletions(-) delete mode 100644 flex/interactive/docker/Makefile delete mode 100644 flex/interactive/docker/README.md delete mode 100644 flex/interactive/docker/interactive-base.Dockerfile rename flex/interactive/docker/interactive-runtime.Dockerfile => k8s/dockerfiles/flex-interactive.Dockerfile (96%) rename flex/interactive/docker/entrypoint.sh => k8s/dockerfiles/interactive-entrypoint.sh (100%) diff --git a/flex/interactive/docker/Makefile b/flex/interactive/docker/Makefile deleted file mode 100644 index b55f4cd832b0..000000000000 --- a/flex/interactive/docker/Makefile +++ /dev/null @@ -1,39 +0,0 @@ -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -WORKING_DIR := $(dir $(MKFILE_PATH)) -DOCKERFILES_DIR := $(WORKING_DIR) -SHORT_SHA := $(shell git rev-parse --short HEAD) - -ifeq ($(REGISTRY),) - REGISTRY := registry.cn-hongkong.aliyuncs.com -endif - -# x86_64 or aarch64 -ARCH := $(shell uname -m) - -VERSION ?= latest - -BUILD_PROGRESS ?= auto -ENABLE_COORDINATOR ?= false - -.PHONY: all interactive-base interactive-runtime - -interactive-base: - cd $(WORKING_DIR) && \ - docker buildx build --platform linux/amd64,linux/arm64 \ - -f interactive-base.Dockerfile \ - -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive-base:${VERSION}-${ARCH} --push . - -interactive-runtime: - cd $(WORKING_DIR) && \ - cd ../../../ && \ - docker build \ - -f ${WORKING_DIR}/interactive-runtime.Dockerfile \ - --target final_image \ - --build-arg ARCH=$(ARCH) \ - --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ - -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-${ARCH} . - -hqps-server-base: - docker build \ - -f $(WORKING_DIR)/Dockerfile \ - -t registry-vpc.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:${VERSION} . diff --git a/flex/interactive/docker/README.md b/flex/interactive/docker/README.md deleted file mode 100644 index 1c8dc14894c8..000000000000 --- a/flex/interactive/docker/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Build Interactive Docker Image - -## Build Base Image - -```bash -make interactive-base -# will be interactive-base to a multi-platform images, amd64 and arm64, and push to the remote -``` - -## Build Runtime Image with latest GraphScope - -```bash -# get the current commit id -commit_id=$(git rev-parse --short HEAD) -# on a arm machine -make interactive-runtime VERSION=${commit_id} # build and push to the remote -# on a x86 machine -make interactive-runtime VERSION=${commit_id} # build and push to the remote -``` - -## Create and Push Docker Manifest - -```bash -VERSION=${real_tag} -sudo docker manifest create registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${VERSION} --amend registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${commit_id}-x86_64 --amend registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${commit_id}-aarch64 - -sudo docker manifest push registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${VERSION} -``` - -## Start Interactive Service - -The following command with start the interactive service on the default modern graph. -```bash -docker run -it -d -p 7687:7687 -p 10000:10000 -p 7777:7777 registry.cn-hongkong.aliyuncs.com/graphscope/interactive:latest -``` \ No newline at end of file diff --git a/flex/interactive/docker/interactive-base.Dockerfile b/flex/interactive/docker/interactive-base.Dockerfile deleted file mode 100644 index fa5bfbd22320..000000000000 --- a/flex/interactive/docker/interactive-base.Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM ubuntu:22.04 - -ENV DEBIAN_FRONTEND=noninteractive - -# change bash as default -SHELL ["/bin/bash", "-c"] - -# g++ + jre 500MB -RUN apt-get update && apt-get -y install sudo locales g++ cmake openjdk-11-jre-headless && \ - locale-gen en_US.UTF-8 && apt-get clean -y && rm -rf /var/lib/apt/lists/* - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# shanghai zoneinfo -ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN apt-get update && apt-get install -y protobuf-compiler libprotobuf-dev maven git vim curl \ - wget python3 make libc-ares-dev doxygen python3-pip net-tools curl default-jdk nlohmann-json3-dev \ - libgoogle-glog-dev libopenmpi-dev libboost-all-dev libyaml-cpp-dev libcrypto++-dev openssl libcurl4-openssl-dev && \ - apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# install libgrape-lite -RUN cd /tmp && \ - git clone https://github.com/alibaba/libgrape-lite.git -b v0.3.2 --single-branch && cd libgrape-lite && \ - mkdir build && cd build && cmake .. -DBUILD_LIBGRAPELITE_TESTS=OFF -DCMAKE_INSTALL_PREFIX=/opt/flex && make -j && make install && rm -rf /tmp/libgrape-lite - -RUN apt-get update && apt-get -y install pkg-config ninja-build xfslibs-dev ragel libpciaccess-dev libxml2-dev libgnutls28-dev liblz4-dev libsctp-dev systemtap-sdt-dev stow libfmt-dev valgrind && \ - apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# install hiactor -RUN cd /tmp && git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch && cd hiactor && \ - git submodule update --init --recursive && ./seastar/seastar/install-dependencies.sh && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/opt/flex -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. && \ - make -j && make install && rm -rf /tmp/hiactor - -# Add graphscope user with user id 1001 -RUN useradd -m graphscope -u 1001 && \ - echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -RUN chown -R graphscope:graphscope /opt/flex - -# set home to graphscope user -ENV HOME=/home/graphscope -USER graphscope -WORKDIR /home/graphscope diff --git a/k8s/Makefile b/k8s/Makefile index dc7491925bb7..e58d557fee82 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -10,7 +10,7 @@ endif ARCH := $(shell uname -m) VERSION ?= latest -VINEYARD_VERSION ?= v0.22.0 +VINEYARD_VERSION ?= v0.23.0 # This is the version of builder base image in most cases, except for graphscope-dev BUILDER_VERSION ?= $(VINEYARD_VERSION) # This is the version of runtime base image @@ -150,6 +150,19 @@ interactive-experimental: -t graphscope/interactive-experimental:${VERSION} \ -f $(DOCKERFILES_DIR)/interactive-experimental.Dockerfile . + +# flex interactive +flex-interactive: + cd $(WORKING_DIR)/.. && \ + docker build \ + -f ${WORKING_DIR}/flex-interactive.Dockerfile \ + --target runtime \ + --build-arg REGISTRY=$(REGISTRY) \ + --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg ARCH=${ARCH} \ + --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ + -t graphscope/interactive:${VERSION} . + learning: cd $(WORKING_DIR)/.. && \ docker build \ diff --git a/flex/interactive/docker/interactive-runtime.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile similarity index 96% rename from flex/interactive/docker/interactive-runtime.Dockerfile rename to k8s/dockerfiles/flex-interactive.Dockerfile index 516edd8f57bc..ce3eca42509d 100644 --- a/flex/interactive/docker/interactive-runtime.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -1,8 +1,10 @@ -ARG ARCH=x86_64 -FROM registry.cn-hongkong.aliyuncs.com/graphscope/interactive-base:v0.0.4 AS builder +# Coordinator of graphscope engines -ARG ARCH +ARG REGISTRY=registry.cn-hongkong.aliyuncs.com +ARG BUILDER_VERSION=latest +ARG ARCH=x86_64 ARG ENABLE_COORDINATOR="false" +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder # change bash as default SHELL ["/bin/bash", "-c"] @@ -55,7 +57,7 @@ RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ ########################### RUNTIME IMAGE ########################### -from ubuntu:22.04 as final_image +from ubuntu:22.04 as runtime ARG ARCH ARG ENABLE_COORDINATOR="false" @@ -90,7 +92,7 @@ COPY --from=builder /opt/flex /opt/flex RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ COPY --from=builder /home/graphscope/GraphScope/flex/interactive/examples/modern_graph/* /opt/flex/share/gs_interactive_default_graph/ COPY --from=builder /home/graphscope/GraphScope/flex/tests/hqps/engine_config_test.yaml /opt/flex/share/engine_config.yaml -COPY --from=builder /home/graphscope/GraphScope/flex/interactive/docker/entrypoint.sh /opt/flex/bin/entrypoint.sh +COPY --from=builder /home/graphscope/GraphScope/k8s/dockerfiles/interactive-entrypoint.sh /opt/flex/bin/entrypoint.sh COPY --from=builder /home/graphscope/GraphScope/flex/third_party/nlohmann-json/single_include/* /opt/flex/include/ RUN sed -i 's/name: modern_graph/name: gs_interactive_default_graph/g' /opt/flex/share/gs_interactive_default_graph/graph.yaml # change the default graph name. diff --git a/flex/interactive/docker/entrypoint.sh b/k8s/dockerfiles/interactive-entrypoint.sh similarity index 100% rename from flex/interactive/docker/entrypoint.sh rename to k8s/dockerfiles/interactive-entrypoint.sh diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 9dc58dc73867..9d59411c046a 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -126,7 +126,7 @@ def interactive(app, graphscope_repo): if graphscope_repo is None: graphscope_repo = default_graphscope_repo_path interactive_build_dir = os.path.join( - graphscope_repo, "flex", "interactive", "docker" + graphscope_repo, "k8s" ) if not os.path.exists(interactive_build_dir) or not os.path.isdir( interactive_build_dir @@ -136,7 +136,7 @@ def interactive(app, graphscope_repo): fg="red", ) return - cmd = ["make", "interactive-runtime", "ENABLE_COORDINATOR=true"] + cmd = ["make", "flex-interactive", "ENABLE_COORDINATOR=true"] run_shell_cmd(cmd, os.path.join(graphscope_repo, interactive_build_dir)) diff --git a/python/graphscope/gsctl/scripts/make_image_command.sh b/python/graphscope/gsctl/scripts/make_image_command.sh index d83ca19b649f..fe546bc5f8ba 100644 --- a/python/graphscope/gsctl/scripts/make_image_command.sh +++ b/python/graphscope/gsctl/scripts/make_image_command.sh @@ -71,6 +71,12 @@ make_interactive-executor() { make interactive-executor REGISTRY=${registry} VERSION=${tag} } +# make the new flex interactive +make_flex-interactive() { + cd k8s + make flex-interactive REGISTRY=${registry} VERSION=${tag} +} + make_learning() { cd k8s make learning REGISTRY=${registry} VERSION=${tag} From 2513f2cb614b9bbcb0c553888111aecbe3081a92 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 6 Aug 2024 19:41:19 +0800 Subject: [PATCH 03/10] fix building of interactive Committed-by: xiaolei.zl from Dev container --- flex/interactive/README.md | 61 +- flex/interactive/bin/gs_interactive | 2220 ------------------- flex/interactive/conf/engine_config.yaml | 12 - flex/interactive/conf/interactive.yaml | 3 - k8s/Makefile | 2 +- k8s/dockerfiles/flex-interactive.Dockerfile | 5 + 6 files changed, 7 insertions(+), 2296 deletions(-) delete mode 100755 flex/interactive/bin/gs_interactive delete mode 100644 flex/interactive/conf/engine_config.yaml delete mode 100755 flex/interactive/conf/interactive.yaml diff --git a/flex/interactive/README.md b/flex/interactive/README.md index 93a1e3c4ebe4..1c130b5c3539 100755 --- a/flex/interactive/README.md +++ b/flex/interactive/README.md @@ -1,63 +1,4 @@ # GraphScope Interactive GraphScope Interactive is a specialized construction of [GraphScope Flex](https://github.com/alibaba/GraphScope/tree/main/flex), designed to handle concurrent graph queries at an impressive speed. Its primary goal is to process as many queries as possible within a given timeframe, emphasizing a high query throughput rate. -For the full documentation of GraphScope Interactive, please refer to [GraphScope-Interactive](https://graphscope.io/docs/interactive_engine/graphscope_interactive). - -## Minimal tutorial - -In this minimal tutorial, we will show you how to run graph service on builtin modern graph. - -### Preparation - -Set `location` to `/home/graphscope/default_graph`. - -### init database - -```bash -./bin/gs_interactive init -c ./conf/interactive.yaml -``` - -### Start service - -```bash -./bin/gs_interactive service start -``` - -### Stop service -```bash -./bin/gs_interactive service stop -``` - -### Restart service -```bash -./bin/gs_interactive service restart -``` - -### Get service status -```bash -./bin/gs_interactive service status -``` - -### Compile stored procedure -```bash -./bin/gs_interactive procedure compile -g modern -i ./examples/modern_graph/count_vertex_num.cypher -``` - -### Disable stored procedure -```bash -./bin/gs_interactive procedure disable -g modern -n count_vertex_num -``` - -### Enable stored procedure -```bash -./bin/gs_interactive procedure enable -g modern -n count_vertex_num -``` - -### Use user defined graph -```bash -./bin/gs_interactive service stop -./bin/gs_interactive database remove -g modern -./bin/gs_interactive database create -g test -c ./examples/modern_graph/graph.yaml -./bin/gs_interactive database import -g test -c ./examples/modern_graph/bulk_load.yaml -./bin/gs_interactive service start -g test -``` \ No newline at end of file +For the full documentation of GraphScope Interactive, please refer to [GraphScope Interactive Documentation](https://graphscope.io/docs/latest/flex/interactive_intro). diff --git a/flex/interactive/bin/gs_interactive b/flex/interactive/bin/gs_interactive deleted file mode 100755 index 6b9043ee0fc3..000000000000 --- a/flex/interactive/bin/gs_interactive +++ /dev/null @@ -1,2220 +0,0 @@ -#!/bin/bash -# Copyright 2020 Alibaba Group Holding Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# The product name -DB_PROD_NAME="interactive" - -# colored error and info functions to wrap messages. -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -NC='\033[0m' # No Color -err() { - echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2 -} - -info() { - echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" -} - -emph(){ - echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" -} - -################## Some Util Functions ################## - -############################################################################### -# -# source: https://github.com/mrbaseman/parse_yaml.git -# -############################################################################### -# Parses a YAML file and outputs variable assignments. Can optionally accept a -# variable name prefix and a variable name separator -# -# Usage: -# parse_yaml file [prefix] [separator] -############################################################################### - -function parse_yaml { - unset i - unset fs - local prefix=$2 - local separator=${3:-_} - - local indexfix=-1 - # Detect awk flavor - if awk --version 2>&1 | grep -q "GNU Awk" ; then - # GNU Awk detected - indexfix=-1 - elif awk -Wv 2>&1 | grep -q "mawk" ; then - # mawk detected - indexfix=0 - fi - - local s='[[:space:]]*' sm='[ \t]*' w='[a-zA-Z0-9_]*' fs=${fs:-$(echo @|tr @ '\034')} i=${i:- } - - ############################################################################### - # cat: read the yaml file into the stream - # awk 1: process multi-line text - # sed 1: remove comments and empty lines - # sed 2: process lists - # sed 3: process dictionaries - # sed 4: rearrange anchors - # sed 5: remove '---'/'...'/quotes, add file separator to create fields for awk 2 - # awk 2: convert the formatted data to variable assignments - ############################################################################### - - cat $1 | \ - awk -F$fs "{multi=0; - if(match(\$0,/$sm\|$sm$/)){multi=1; sub(/$sm\|$sm$/,\"\");} - if(match(\$0,/$sm>$sm$/)){multi=2; sub(/$sm>$sm$/,\"\");} - while(multi>0){ - str=\$0; gsub(/^$sm/,\"\", str); - indent=index(\$0,str); - indentstr=substr(\$0, 0, indent+$indexfix) \"$i\"; - obuf=\$0; - getline; - while(index(\$0,indentstr)){ - obuf=obuf substr(\$0, length(indentstr)+1); - if (multi==1){obuf=obuf \"\\\\n\";} - if (multi==2){ - if(match(\$0,/^$sm$/)) - obuf=obuf \"\\\\n\"; - else obuf=obuf \" \"; - } - getline; - } - sub(/$sm$/,\"\",obuf); - print obuf; - multi=0; - if(match(\$0,/$sm\|$sm$/)){multi=1; sub(/$sm\|$sm$/,\"\");} - if(match(\$0,/$sm>$sm$/)){multi=2; sub(/$sm>$sm$/,\"\");} - } - print}" | \ - sed -e "s|^\($s\)?|\1-|" \ - -ne "s|^\($s\)-$s\($w\)$s:$s\(.*\)|\1-\n\1 \2: \3|" \ - -ne "s|^$s#.*||;s|$s#[^\"']*$||;s|^\([^\"'#]*\)#.*|\1|;t 1" \ - -ne "t" \ - -ne ":1" \ - -ne "s|^$s\$||;t 2" \ - -ne "p" \ - -ne ":2" \ - -ne "d" | \ - sed -ne "s|,$s\]|]|g" \ - -e ":1" \ - -e "s|^\($s\)\($w\)$s:$s\(&$w\)$s\[$s\(.*\)$s,$s\(.*\)$s\]|\1\2: \3[\4]\n\1$i- \5|;t 1" \ - -e "s|^\($s\)\($w\)$s:$s\(&$w\)$s\[$s\(.*\)$s\]|\1\2: \3\n\1$i- \4|;" \ - -e ":2" \ - -e "s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s,$s\(.*\)$s\]|\1\2: [\3]\n\1$i- \4|;t 2" \ - -e "s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s\]|\1\2:\n\1$i- \3|;" \ - -e ":3" \ - -e "s|^\($s\)-$s\[$s\(.*\)$s,$s\(.*\)$s\]|\1- [\2]\n\1$i- \3|;t 3" \ - -e "s|^\($s\)-$s\[$s\(.*\)$s\]|\1-\n\1$i- \2|;p" | \ - sed -ne "s|,$s}|}|g" \ - -e ":1" \ - -e "s|^\($s\)-$s{$s\(.*\)$s,$s\($w\)$s:$s\(.*\)$s}|\1- {\2}\n\1$i\3: \4|;t 1" \ - -e "s|^\($s\)-$s{$s\(.*\)$s}|\1-\n\1$i\2|;" \ - -e ":2" \ - -e "s|^\($s\)\($w\)$s:$s\(&$w\)$s{$s\(.*\)$s,$s\($w\)$s:$s\(.*\)$s}|\1\2: \3 {\4}\n\1$i\5: \6|;t 2" \ - -e "s|^\($s\)\($w\)$s:$s\(&$w\)$s{$s\(.*\)$s}|\1\2: \3\n\1$i\4|;" \ - -e ":3" \ - -e "s|^\($s\)\($w\)$s:$s{$s\(.*\)$s,$s\($w\)$s:$s\(.*\)$s}|\1\2: {\3}\n\1$i\4: \5|;t 3" \ - -e "s|^\($s\)\($w\)$s:$s{$s\(.*\)$s}|\1\2:\n\1$i\3|;p" | \ - sed -e "s|^\($s\)\($w\)$s:$s\(&$w\)\(.*\)|\1\2:\4\n\3|" \ - -e "s|^\($s\)-$s\(&$w\)\(.*\)|\1- \3\n\2|" | \ - sed -ne "s|^\($s\):|\1|" \ - -e "s|^\($s\)\(---\)\($s\)||" \ - -e "s|^\($s\)\(\.\.\.\)\($s\)||" \ - -e "s|^\($s\)-$s[\"']\(.*\)[\"']$s\$|\1$fs$fs\2|p;t" \ - -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p;t" \ - -e "s|^\($s\)-$s\(.*\)$s\$|\1$fs$fs\2|" \ - -e "s|^\($s\)\($w\)$s:$s[\"']\?\(.*\)$s\$|\1$fs\2$fs\3|" \ - -e "s|^\($s\)[\"']\?\([^&][^$fs]\+\)[\"']$s\$|\1$fs$fs$fs\2|" \ - -e "s|^\($s\)[\"']\?\([^&][^$fs]\+\)$s\$|\1$fs$fs$fs\2|" \ - -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)$s\$|\1$fs\2$fs\3|" \ - -e "s|^\($s\)[\"']\([^&][^$fs]*\)[\"']$s\$|\1$fs$fs$fs\2|" \ - -e "s|^\($s\)[\"']\([^&][^$fs]*\)$s\$|\1$fs$fs$fs\2|" \ - -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|" \ - -e "s|^\($s\)\([^&][^$fs]*\)[\"']$s\$|\1$fs$fs$fs\2|" \ - -e "s|^\($s\)\([^&][^$fs]*\)$s\$|\1$fs$fs$fs\2|" \ - -e "s|$s\$||p" | \ - awk -F$fs "{ - gsub(/\t/,\" \",\$1); - if(NF>3){if(value!=\"\"){value = value \" \";}value = value \$4;} - else { - if(match(\$1,/^&/)){anchor[substr(\$1,2)]=full_vn;getline}; - indent = length(\$1)/length(\"$i\"); - vname[indent] = \$2; - value= \$3; - for (i in vname) {if (i > indent) {delete vname[i]; idx[i]=0}} - if(length(\$2)== 0){ vname[indent]= ++idx[indent] }; - vn=\"\"; for (i=0; i0)&&index(val, ref)==1){ - tmpval=assignment[val]; - sub(ref,full_vn,val); - if(match(val,\"$separator\$\")){ - gsub(ref,full_vn,tmpval); - } else if (length(tmpval) > 0) { - printf(\"%s=\\\"%s\\\"\n\", val, tmpval); - } - assignment[val]=tmpval; - } - } - } - } else if (length(value) > 0) { - printf(\"%s=\\\"%s\\\"\n\", full_vn, value); - } - }END{ - for(val in assignment){ - if(match(val,\"$separator\$\")) - printf(\"%s=\\\"%s\\\"\n\", val, assignment[val]); - } - }" -} - -# check if the file exists, if not, exit. -function check_file_exists(){ - if [ ! -f "$1" ]; then - err "file $1 not exists" - exit 1 - fi -} -function check_directory_exists(){ - if [ ! -d "$1" ]; then - err "directory $1 not exists" - exit 1 - fi -} - -HOST_DB_HOME="$( - cd "$(dirname "$0")/../" >/dev/null 2>&1 - pwd -P -)" -info "HOST_DB_HOME = ${HOST_DB_HOME}" -mkdir -p "${HOST_DB_HOME}/data" - -################### GET USER INFO ################### -# get uid -uid=$(id -u) -# get group id -gid=$(id -g) - - -#################### DEFINE CONSTANTS #################### - -# the log directory -# HOST_DB_INTERACTIVE_YAML="${HOST_DB_CONF_DIR}/interactive.yaml" -HOST_DB_RUNNING_FILE="${HOST_DB_HOME}/.running" -HOST_DB_ENV_FILE="${HOST_DB_HOME}/.env" - -DOCKER_DB_GRAPHSCOPE_HOME="/opt/flex" -DOCKER_DB_GIE_HOME="${DOCKER_DB_GRAPHSCOPE_HOME}" -DOCKER_DB_SERVER_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/interactive_server" -DOCKER_DB_GRAPH_IMPORT_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/bulk_loader" -DOCKER_DB_COMPILER_BIN="com.alibaba.graphscope.GraphServer" -DOCKER_DB_GEN_BIN="${DOCKER_DB_GRAPHSCOPE_HOME}/bin/load_plan_and_gen.sh" -HOST_DB_TMP_DIR="/tmp/" - -#################### DEFINE DEFAULT CONSTANTS #################### -DATABASE_VERSION="v0.3" -DATABASE_DEFAULT_GRAPH_NAME="gs_interactive_default_graph" -DATABASE_CURRENT_GRAPH_NAME=${DATABASE_DEFAULT_GRAPH_NAME} -DATABASE_DEFAULT_GRAPH_DOCKER_PATH="/home/graphscope/${DATABASE_DEFAULT_GRAPH_NAME}" -DATABASE_DEFAULT_GRAPH_MOUNT_CMD="${HOST_DB_HOME}/examples/modern_graph/:${DATABASE_DEFAULT_GRAPH_DOCKER_PATH}" -DATABASE_VOLUMES="${DATABASE_DEFAULT_GRAPH_MOUNT_CMD}" -DATABASE_LOG_LEVEL="INFO" -DATABASE_PORTS="" - -## compiler related default configuration -DATABASE_COMPILER_PLANNER_IS_ON="true" -DATABASE_COMPILER_PLANNER_OPT="RBO" -DATABASE_COMPILER_PLANNER_RULES="FilterIntoJoinRule,FilterMatchRule,NotMatchToAntiJoinRule" -DATABASE_COMPILER_ENDPOINT_ADDRESS="localhost" -DATABASE_COMPILER_BOLT_DISABLED="false" -DATABASE_COMPILER_BOLT_PORT="7687" -DATABASE_COMPILER_GREMLIN_DISABLED="true" -DATABASE_COMPILER_GREMLIN_PORT="8182" -DATABASE_COMPILER_QUERY_TIMEOUT="20000" - -## hiactor related default configuration -DATABASE_COMPUTE_ENGINE_PORT="10000" -DATABASE_ADMIN_PORT="7777" -DATABASE_COMPUTE_ENGINE_THREAD_NUM_PER_WORKER=1 - -## directories -DATABASE_WORKSPACE="/home/graphscope/workspace/" -DATABASE_DATA_DIR_NAME="data" -DATABASE_LOG_DIR_NAME="logs" -DATABASE_CONF_DIR_NAME="conf" - - -################### IMAGE VERSION ################### -GIE_DB_IMAGE_NAME="registry.cn-hongkong.aliyuncs.com/graphscope/${DB_PROD_NAME}" -#GIE_DB_IMAGE_NAME="interactive" -GIE_DB_CONTAINER_NAME="${DB_PROD_NAME}-server" - - -#################### Prepare uncreated directories #################### - -info "Finish create log dir" - -#################### DEFINE FUNCTIONS #################### - -function check_graph_name_valid() { - if [ $# -ne 1 ]; then - err "Expect graph name given." - exit 1 - fi - local graph_name=$1 - # check graph_name can be a valid directory name - if [[ ! "${graph_name}" =~ ^[a-zA-Z0-9_]+$ ]]; then - err "graph name [${graph_name}] is not valid, only [a-zA-Z0-9_] are allowed." - exit 1 - fi -} -function check_running_containers_and_exit(){ - # check if there is any running containers - info "Check running containers and exit" - running_containers=$(docker ps -a --format "{{.Names}}" | grep "${GIE_DB_CONTAINER_NAME}") - if [[ -n "${running_containers}" ]]; then - err "There are running containers: ${running_containers}, please stop them first." - exit 1 - fi - info "finish check" -} - -function check_container_running(){ - if [ "$(docker inspect -f '{{.State.Running}}' "${GIE_DB_CONTAINER_NAME}")" = "true" ]; then - info "container ${GIE_DB_CONTAINER_NAME} is running" - else - info "container ${GIE_DB_CONTAINER_NAME} is not running" - # start the container - docker start "${GIE_DB_CONTAINER_NAME}" - fi -} - -function ensure_container_running(){ - if [ "$(docker inspect -f '{{.State.Running}}' "${GIE_DB_CONTAINER_NAME}")" = "true" ]; then - info "container ${GIE_DB_CONTAINER_NAME} is running" - else - info "container ${GIE_DB_CONTAINER_NAME} is not running" - # start the container - docker start "${GIE_DB_CONTAINER_NAME}" - fi -} - -function check_process_running_in_container(){ - local container_name=$1 - local process_name=$2 - local error_msg=$3 - local process_id=$(docker top "${container_name}" | grep "${process_name}" | awk '{print $2}') - if [ -z "${process_id}" ]; then - err "process ${process_name} is not running in container ${container_name}" - err "${error_msg}" - exit 1 - fi - info "process ${process_name} is running in container ${container_name}, process id is ${process_id}" -} - -function check_process_not_running_in_container(){ - local container_name=$1 - local process_name=$2 - local error_msg=$3 - local process_id=$(docker top "${container_name}" | grep "${process_name}" | awk '{print $2}') - if [ -z "${process_id}" ]; then - info "process ${process_name} is not running in container ${container_name}" - else - err "process ${process_name} is running in container ${container_name}, process id is ${process_id}" - err "${error_msg}" - exit 1 - fi -} - -# check the given graph is locked or not. -function check_graph_not_running(){ - info "Check graph whether is not running" - if [ $# -ne 1 ]; then - err "Expect graph name given." - exit 1 - fi - local graph_name=$1 - # check whether .lock is presented in container's data/${graph_name}/ directory - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - local lock_file="${DATABASE_WORKSPACE}/data/${graph_name}/.lock" - info "Check lock file ${lock_file}" - # check lock_file whether exists in container, if not exists, exit 0, else exit 1 - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ ! -f ${lock_file} ]" -} - -function update_init_config_from_yaml(){ - if [ $# -ne 1 ]; then - err "Expect configuration file given" - exit 1 - fi - config_file=$1 - eval $(parse_yaml "${config_file}") - # update workspace if exists - if [[ -n "${workspace}" ]]; then - DATABASE_WORKSPACE="${workspace}" - fi - # update database version if exists - if [[ -n "${version}" ]]; then - DATABASE_VERSION="${version}" - fi - # append the found volumes to DATABASE_VOLUMES - # map the HOST_DB_HOME/data/ to ${DATABASE_WORKSPACE}/data - DATABASE_VOLUMES="${DATABASE_VOLUMES},${HOST_DB_HOME}/data:${DATABASE_WORKSPACE}/data" - - x=1 - while true; do - volume_x_key="volume_${x}" - volume_x=$(eval echo "\$${volume_x_key}") - if [ -z "${volume_x}" ]; then - break - fi - DATABASE_VOLUMES="${DATABASE_VOLUMES},${volume_x}" - x=$((x + 1)) - done - # append compiler port and engine port to DATABASE_PORTS - if [ "${DATABASE_COMPILER_BOLT_DISABLED}" == "false" ]; then - DATABASE_PORTS="${DATABASE_COMPILER_BOLT_PORT}:${DATABASE_COMPILER_BOLT_PORT}" - fi - if [ "${DATABASE_COMPILER_GREMLIN_DISABLED}" == "false" ]; then - DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_COMPILER_GREMLIN_PORT}:${DATABASE_COMPILER_GREMLIN_PORT}" - fi - DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_COMPUTE_ENGINE_PORT}:${DATABASE_COMPUTE_ENGINE_PORT}" - DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_ADMIN_PORT}:${DATABASE_ADMIN_PORT}" -} - -function update_engine_config_from_yaml(){ - if [ $# -ne 1 ]; then - err "Expect configuration file given" - exit 1 - fi - config_file=$1 - eval $(parse_yaml "${config_file}") - if [[ -n "${log_level}" ]]; then - DATABASE_LOG_LEVEL="${log_level}" - fi - # default_graph - if [[ -n "${default_graph}" ]]; then - DATABASE_CURRENT_GRAPH_NAME="${default_graph}" - fi - # update hiactor shard num - if [[ -n "${compute_engine_thread_num_per_worker}" ]]; then - DATABASE_COMPUTE_ENGINE_THREAD_NUM_PER_WORKER="${compute_engine_thread_num_per_worker}" - fi - # compiler - if [[ -n ${compiler_planner_is_on} ]]; then - DATABASE_COMPILER_PLANNER_IS_ON="${compiler_planner_is_on}" - fi - info "Found compiler planner opt: ${compiler_planner_is_on}, ${DATABASE_COMPILER_PLANNER_IS_ON}" - if [[ -n ${compiler_planner_opt} ]]; then - DATABASE_COMPILER_PLANNER_OPT="${compiler_planner_opt}" - fi - # append the founded compiler planner rules to DATABASE_COMPILER_PLANNER_RULES - x=1 - CURRENT_DATABASE_COMPILER_PLANNER_RULES="" - while true; do - compiler_planner_rules_x_key="compiler_planner_rules_${x}" - compiler_planner_rules_x=$(eval echo "\$${compiler_planner_rules_x_key}") - if [ -z "${compiler_planner_rules_x}" ]; then - break - fi - # check compiler_planner_rules_x present in DATABASE_COMPILER_PLANNER_RULES, if not, append - if [[ ! "${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" =~ "${compiler_planner_rules_x}" ]]; then - CURRENT_DATABASE_COMPILER_PLANNER_RULES="${CURRENT_DATABASE_COMPILER_PLANNER_RULES},${compiler_planner_rules_x}" - fi - x=$((x + 1)) - done - # if CURRENT_DATABASE_COMPILER_PLANNER_RULES is not empty,override DATABASE_COMPILER_PLANNER_RULES - if [[ -n "${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" ]]; then - # remove the first ',' - CURRENT_DATABASE_COMPILER_PLANNER_RULES=$(echo "${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" | sed 's/^,//g') - DATABASE_COMPILER_PLANNER_RULES="${CURRENT_DATABASE_COMPILER_PLANNER_RULES}" - fi - if [[ -n "${compiler_endpoint_address}" ]]; then - DATABASE_COMPILER_ENDPOINT_ADDRESS="${compiler_endpoint_address}" - fi - if [[ -n "${compiler_endpoint_bolt_connector_port}" ]]; then - DATABASE_COMPILER_BOLT_PORT="${compiler_endpoint_bolt_connector_port}" - fi - if [[ -n "${compiler_endpoint_bolt_connector_disabled}" ]]; then - DATABASE_COMPILER_BOLT_DISABLED="${compiler_endpoint_bolt_connector_disabled}" - fi - if [[ -n "${compiler_endpoint_gremlin_connector_port}" ]]; then - DATABASE_COMPILER_GREMLIN_PORT="${compiler_endpoint_gremlin_connector_port}" - fi - if [[ -n "${compiler_endpoint_gremlin_connector_disabled}" ]]; then - DATABASE_COMPILER_GREMLIN_DISABLED="${compiler_endpoint_gremlin_connector_disabled}" - fi - if [[ -n "${compiler_query_timeout}" ]]; then - DATABASE_COMPILER_QUERY_TIMEOUT="${compiler_query_timeout}" - fi -} - -function amplify_graph_schema(){ - # two args, input schema_file and output schema_file - if [ $# -ne 2 ]; then - err "Expect two args, but got $#" - exit 1 - fi - input_schema_file=$1 - output_schema_file=$2 - # check input_schema_file exists - check_file_exists "${input_schema_file}" - # rm output_schema_file if exists - if [ -f "${output_schema_file}" ]; then - rm "${output_schema_file}" - fi - # read input_schema_file line by line, and append to output_schema_file - local cur_vertex_label_ind=0 - local cur_edge_label_ind=0 - local cur_prop_ind=0 - # the type id may not be specified in input_schema_file - local type_id_specified=false - local prop_id_specified=false - # if stored_type is not specified in input_schema_file's content, append it to output_schema_file - stored_type_specified=false - label_type="vertex" - IFS='' - while read -r line; do - if [[ "${line}" =~ ^[[:space:]]*# ]]; then - echo "found comment line ${line}" - else - # if line contains 'type_name', then prepend - type_id: 0 before it - if [[ "${line}" =~ vertex_types ]]; then - label_type="vertex" - echo ${line} >> "${output_schema_file}" - elif [[ "${line}" =~ edge_types ]]; then - label_type="edge" - echo ${line} >> "${output_schema_file}" - elif [[ "${line}" =~ type_id ]]; then - type_id_specified=true - echo ${line} >> "${output_schema_file}" - elif [[ "${line}" =~ property_id ]]; then - prop_id_specified=true - echo ${line} >> "${output_schema_file}" - elif [[ "${line}" =~ type_name ]] && [ "${type_id_specified}" == "false" ]; then - if [ "${label_type}" == "edge" ]; then - echo " - type_id: ${cur_edge_label_ind}" >> "${output_schema_file}" - cur_edge_label_ind=$((cur_edge_label_ind + 1)) - else - echo " - type_id: ${cur_vertex_label_ind}" >> "${output_schema_file}" - cur_vertex_label_ind=$((cur_vertex_label_ind + 1)) - fi - cur_prop_ind=0 - type_id_specified=false - echo "${line}" | sed 's/- / /g' >> "${output_schema_file}" - elif [[ "${line}" =~ property_name ]] && [ "${prop_id_specified}" == "false" ]; then - echo " - property_id: ${cur_prop_ind}" >> "${output_schema_file}" - cur_prop_ind=$((cur_prop_ind + 1)) - prop_id_specified=false - # replace the '- ' in ${line} and append to output_schema_file - echo "${line}" | sed 's/- / /g' >> "${output_schema_file}" - elif [[ "${line}" =~ store_type ]]; then - stored_type_specified=true - emph "stored type : ${stored_type_specified}" - echo "${line}" >> "${output_schema_file}" - else - echo "${line}" >> "${output_schema_file}" - fi - fi - done <<< $(cat "${input_schema_file}") - if [ "${stored_type_specified}" == "false" ]; then - emph "stored type : ${stored_type_specified}" - echo "store_type: mutable_csr" >> "${output_schema_file}" - fi - info "Finish amplify schema file ${input_schema_file} to ${output_schema_file}" -} - - -#################### DEFINE USAGE #################### - -function init_usage() { - cat << EOF - gs_interactive init -c [--config] - Init the database, create the containers. Specify the database version and volume mounting in the config yaml. -EOF -} - -function destroy_usage() { - cat << EOF - gs_interactive destroy - Destroy the current database, remove the container. -EOF -} - -function create_usage() { - cat << EOF - gs_interactive database create -g [--graph] -c [--config] - Create a graph in database, with the provided schema file. - User should import data to the created graph. -EOF -} - -function remove_usage() { - cat << EOF - gs_interactive database remove -g [--graph] - Remove the database with the given graph. -EOF -} - -function import_usage() { - cat << EOF - gs_interactive database import -g [--graph] -c [--config] - Load the raw data specified in bulk load file to the specified graph. -EOF -} - -function list_usage() { - cat << EOF - gs_interactive database list - List all graphs in the database. -EOF -} - -function database_usage(){ - create_usage - remove_usage - import_usage - list_usage -} - - -function start_usage() { - cat << EOF - gs_interactive service start -g [--graph] -c [--config] - Start the graph service on the specified graph, with the provided engine config file. -EOF -} - -function stop_usage() { - cat << EOF - gs_interactive service stop - Stop the database with the given graph. -EOF -} - -function restart_usage() { - cat << EOF - gs_interactive service restart -c [--config] [engine config file] - Restart the database with current running graph. Can update with new engine config file. -EOF -} - -function get_log_usage() { - cat << EOF - gs_interactive service get_log -o [--output] output directory - Get the log of the specified service/compiler, and write to the output file. -EOF -} - -function services_usage(){ - start_usage - stop_usage - restart_usage - get_log_usage -} - -function compile_usage(){ - cat << EOF - gs_interactive procedure compile -g[--graph] -i -d [--description] - --compile_only - Compile cypher/.cc to dynamic library, according to the schema of graph. The output library will be placed at ./data/{graph_name}/lib. - If --compile_only is specified, the library will not be loaded to the graph. -EOF -} - -## .enable and .disable file controls the stored procedure enable/disable - -function enable_proc_usage(){ - cat << EOF - gs_interactive procedure enable -g[--graph] -n[--name] - -c[--config] - Enable the stored procedure in the given graph, with the provided library. - stored_procedures.yaml contains the stored procedure names at each line. -EOF -} - -function disable_proc_usage(){ - cat << EOF - gs_interactive procedure disable -g[--graph] -n[--name] - -c[--config] - Disable the stored procedure in the given graph, with the provided library. - stored_procedures.yaml contains the stored procedure names at each line. -EOF -} - -function show_stored_procedure_usage(){ - cat << EOF - gs_interactive procedure show -g[--graph] graph_name - Show all stored procedure for the given graph. -EOF -} - - -function procedure_usage(){ - compile_usage - enable_proc_usage - disable_proc_usage - show_stored_procedure_usage -} - -# parse the args and set the variables. -function usage() { - init_usage - destroy_usage - database_usage - services_usage - procedure_usage -} - -################### Generate config file ################### -function do_gen_conf(){ - # receive only one args, the config file - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -o | --output) - output_config_file="$2" - shift - shift - ;; - *) - err "unknown option $1" - exit 1 - ;; - esac - done - - #if output_config_file exists, remove the file, if remove failed, exists - if [ -f "${output_config_file}" ]; then - if ! rm "${output_config_file}"; then - err "remove ${output_config_file} failed" - exit 1 - fi - fi - - # echo directories - echo "directories:" >> ${output_config_file} - echo " workspace: ${DATABASE_WORKSPACE}" >> ${output_config_file} - echo " subdirs:" >> ${output_config_file} - echo " data: ${DATABASE_DATA_DIR_NAME}" >> ${output_config_file} - echo " logs: ${DATABASE_LOG_DIR_NAME}" >> ${output_config_file} - echo " conf: ${DATABASE_CONF_DIR_NAME}" >> ${output_config_file} - - # log level - echo "log_level: ${DATABASE_LOG_LEVEL}" >> ${output_config_file} - - # current graph - echo "default_graph: ${DATABASE_CURRENT_GRAPH_NAME}" >> ${output_config_file} - - - #compute_engine - echo "compute_engine:" >> ${output_config_file} - echo " type: hiactor" >> ${output_config_file} - echo " workers:" >> ${output_config_file} - echo " - localhost:${DATABASE_COMPUTE_ENGINE_PORT}" >> ${output_config_file} - echo " thread_num_per_worker: ${DATABASE_COMPUTE_ENGINE_THREAD_NUM_PER_WORKER}" >> ${output_config_file} - - - #compiler - echo "compiler:" >> ${output_config_file} - echo " planner:" >> ${output_config_file} - echo " is_on: ${DATABASE_COMPILER_PLANNER_IS_ON}" >> ${output_config_file} - echo " opt: ${DATABASE_COMPILER_PLANNER_OPT}" >> ${output_config_file} - # split compiler planner rules and put as sequences in yaml - echo " rules:" >> ${output_config_file} - IFS=',' read -ra RULES_ARRAY <<<"${DATABASE_COMPILER_PLANNER_RULES}" - for rule in "${RULES_ARRAY[@]}"; do - echo " - ${rule}" >> ${output_config_file} - done - echo " endpoint:" >> ${output_config_file} - echo " default_listen_address: ${DATABASE_COMPILER_ENDPOINT_ADDRESS}" >> ${output_config_file} - echo " bolt_connector:" >> ${output_config_file} - echo " disabled: ${DATABASE_COMPILER_BOLT_DISABLED}" >> ${output_config_file} - echo " port: ${DATABASE_COMPILER_BOLT_PORT}" >> ${output_config_file} - echo " gremlin_connector:" >> ${output_config_file} - echo " disabled: ${DATABASE_COMPILER_GREMLIN_DISABLED}" >> ${output_config_file} - echo " port: ${DATABASE_COMPILER_GREMLIN_PORT}" >> ${output_config_file} - echo " query_timeout: ${DATABASE_COMPILER_QUERY_TIMEOUT}" >> ${output_config_file} - - #echo http_service - echo "http_service:" >> ${output_config_file} - echo " default_listen_address: localhost" >> ${output_config_file} - echo " admin_port: ${DATABASE_ADMIN_PORT}" >> ${output_config_file} - echo " query_port: ${DATABASE_COMPUTE_ENGINE_PORT}" >> ${output_config_file} - - info "Finish generate config file ${output_config_file}" -} - -function generate_real_engine_conf(){ - # expect less than three args - if [ $# -gt 2 ] || [ $# -eq 0 ]; then - err "Expect one or two args, but got $#" - exit 1 - fi - if [ $# -eq 1 ]; then - real_engine_config_file=$1 - info "engine config file is not specified, using default engine config" - if ! do_gen_conf -o ${real_engine_config_file}; then - err "generate engine config file failed" - exit 1 - fi - else - engine_config_file=$1 - real_engine_config_file=$2 - check_file_exists "${engine_config_file}" - update_engine_config_from_yaml "${engine_config_file}" - if ! do_gen_conf -o ${real_engine_config_file}; then - err "generate engine config file failed" - exit 1 - fi - fi -} - -function update_graph_yaml_with_procedure_enabling(){ - # expect one args, graph name - if [ $# -ne 1 ]; then - err "Expect one args, but got $#" - exit 1 - fi - graph_name=$1 - - # gather .enable and append to graph yaml - - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - # copy graph_name's graph0.yaml to ${HOST_DB_TMP_DIR} - docker cp "${GIE_DB_CONTAINER_NAME}:${DATABASE_WORKSPACE}/data/${graph_name}/graph0.yaml" "${HOST_DB_TMP_DIR}/graph0.yaml" - echo "" >> ${HOST_DB_TMP_DIR}/graph0.yaml - echo "stored_procedures:" >> ${HOST_DB_TMP_DIR}/graph0.yaml - echo " enable_lists:" >> ${HOST_DB_TMP_DIR}/graph0.yaml - # copy graph_name's .enable file to ${HOST_DB_TMP_DIR} - if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then - if ! rm "${HOST_DB_TMP_DIR}/.enable"; then - err "remove ${HOST_DB_TMP_DIR}/.enable failed" - exit 1 - fi - fi - - docker cp "${GIE_DB_CONTAINER_NAME}:${DATABASE_WORKSPACE}/data/${graph_name}/plugins/.enable" "${HOST_DB_TMP_DIR}/.enable" || true - # get .enable and .disable file - local enable_file="${HOST_DB_TMP_DIR}/.enable" - any_stored_procedures=false - # check enable_file and disable_file exists - if [ ! -f "${enable_file}" ]; then - info "enable file ${enable_file} not exists, all stored procedure are enabled" - else - # cat all lines in enable_file and split by '\n' - while IFS= read -r line; do - echo "Found enable procedure: $line" - echo " - ${line}" >> ${HOST_DB_TMP_DIR}/graph0.yaml - any_stored_procedures=true - done < ${enable_file} - fi - if [ "${any_stored_procedures}" = false ]; then - echo " directory: not-a-directory" >> ${HOST_DB_TMP_DIR}/graph0.yaml - else - echo " directory: plugins" >> ${HOST_DB_TMP_DIR}/graph0.yaml - fi - - # copy graph0.yaml to container - docker cp "${HOST_DB_TMP_DIR}/graph0.yaml" "${GIE_DB_CONTAINER_NAME}:${DATABASE_WORKSPACE}/data/${graph_name}/graph.yaml" - info "Finish update graph yaml with procedure enabling, add stored procedures of size ${#ENABLED_ARRAY[@]}." -} - - -function check_database_initialized(){ - # check if the container is running - running_containers=$(docker ps -a --format "{{.Names}}" | grep "${GIE_DB_CONTAINER_NAME}") - if [[ -z "${running_containers}" ]]; then - err "Database is not initialized, please run init command first." - exit 1 - fi -} - -#################### Init database #################### -# Init the current data base. -# create a user with same user id in container -function do_init(){ - # check running containers and exit - check_running_containers_and_exit - info "Ok, no running instance found, start init database..." - # if no containers running, proceed to init - -# check args num 1, and get the first args as CONFIG_FILE - if [ $# -eq 0 ]; then - err "init command need 1 args, but got $#" - init_usage - exit 1 - fi - - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -c | --config) - config_file="$2" - shift # past argument - shift - ;; - *) - err "unknown option $1" - init_usage - exit 1 - ;; - esac - done - - check_file_exists "${config_file}" - - # Parse the configuration presented in yaml, and override the default values. - update_init_config_from_yaml "${config_file}" - - #if $HOST_DB_ENV_FILE exists, remove it - if [ -f "${HOST_DB_ENV_FILE}" ]; then - if ! rm "${HOST_DB_ENV_FILE}"; then - err "remove ${HOST_DB_ENV_FILE} failed" - exit 1 - fi - fi - if ! touch "${HOST_DB_ENV_FILE}"; then - err "create ${HOST_DB_ENV_FILE} failed" - exit 1 - fi - - #0. Found workspace - info "Found docker db home: ${DATABASE_WORKSPACE}" - # put docker_workspace into env - echo "export DATABASE_WORKSPACE=${DATABASE_WORKSPACE}" >> ${HOST_DB_ENV_FILE} - echo "export DATABASE_DATA_DIR_NAME=${DATABASE_DATA_DIR_NAME}" >> ${HOST_DB_ENV_FILE} - info "Found database version: ${DATABASE_VERSION}" - - #2. Found mounting volumes from yaml file - mount_cmd="" - # split DATABASE_VOLUMES and append to mount_cmd - IFS=',' read -ra VOLUME_ARRAY <<<"${DATABASE_VOLUMES}" - for volume in "${VOLUME_ARRAY[@]}"; do - # split with : and check host path exists - volume_value_array=(${volume//:/ }) - # if volume_value_array length is not 2, error - if [ ${#volume_value_array[@]} -ne 2 ]; then - err "volume ${volume_value_array} is not valid, should be :" - exit 1 - fi - # get host_path - host_path=${volume_value_array[0]} - docker_path=${volume_value_array[1]} - # check host_path exists - info "Found host path: ${host_path}" - check_directory_exists "${host_path}" || (err "host path ${host_path} not exists" && exit 1) - mount_cmd="${mount_cmd} -v ${volume}" - done -# mount_cmd="${mount_cmd} -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro" - - info "Found docker volumes: ${mount_cmd}" - - #3. get mapped port - port_cmd="" - # split the DATABASE_PORTS and append to port_cmd - IFS=',' read -ra DATABASE_PORTS_ARRAY <<<"${DATABASE_PORTS}" - for ports in "${DATABASE_PORTS_ARRAY[@]}"; do - port_x_value_array=(${ports//:/ }) - # if volume_x_value_array length is not 2, error - if [ ${#port_x_value_array[@]} -ne 2 ]; then - err "port ${port_x_value_array} is not valid, should be :" - exit 1 - fi - # get host_path - host_port=${port_x_value_array[0]} - docker_port=${port_x_value_array[1]} - #check port are int - if ! [[ "${host_port}" =~ ^[0-9]+$ ]]; then - err "host port ${host_port} is not valid" - exit 1 - fi - if ! [[ "${docker_port}" =~ ^[0-9]+$ ]]; then - err "docker port ${docker_port} is not valid" - exit 1 - fi - port_cmd="${port_cmd} -p ${host_port}:${docker_port}" - done - info "Found docker port: ${port_cmd}" - - # get uid - local uid=$(id -u) - local gid=$(id -g) - # get group name - local group_name=$(id -gn) - # get username - local username=$(id -un) - - GIE_DB_IMAGE_NAME_TAG="${GIE_DB_IMAGE_NAME}:${DATABASE_VERSION}" - cmd="docker run --entrypoint bash -it -d --privileged --name ${GIE_DB_CONTAINER_NAME}" - - # create user in container - cmd="${cmd} ${port_cmd} ${mount_cmd} ${GIE_DB_IMAGE_NAME_TAG}" - - info "Running cmd: ${cmd}" - # eval $cmd, exit if failed - if ! eval $cmd; then - err "Fail to init database." - exit 1 - fi - - info "Finish init database" - - # create the workspace directory in container - docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${DATABASE_WORKSPACE}" || exit 1 - docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "sudo chown -R graphscope:graphscope ${DATABASE_WORKSPACE}" || exit 1 - docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${DATABASE_WORKSPACE}/logs" || exit 1 - docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${DATABASE_WORKSPACE}/conf" || exit 1 - docker exec -u graphscope "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${DATABASE_WORKSPACE}/data" || exit 1 - - # create default_graph - do_create -g ${DATABASE_DEFAULT_GRAPH_NAME} -c ${HOST_DB_HOME}/examples/modern_graph/graph.yaml - # check whether do_import success - if ! do_import -g ${DATABASE_DEFAULT_GRAPH_NAME} -c ${HOST_DB_HOME}/examples/modern_graph/bulk_load.yaml; then - err "Fail to import default graph" - exit 1 - fi - info "Successfully create and import default graph: [${DATABASE_DEFAULT_GRAPH_NAME}]" -} - - -#################### Create graph #################### -function do_create(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift # past argument - shift - ;; - -c | --config) - schema_file="$2" - shift - shift - ;; - *) - err "unknown option $1" - create_usage - exit 1 - ;; - esac - done - # check graph_name is set - if [ -z "${graph_name}" ]; then - err "graph name is not specified" - create_usage - exit 1 - fi - - if [ -z "${schema_file}" ]; then - err "graph schema is not specified, need graph_schema.yaml given by -c option" - create_usage - exit 1 - fi - - check_graph_name_valid "${graph_name}" - if [ -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] has already been created." - exit 1 - fi - check_file_exists "${schema_file}" - amplify_schema_file="${HOST_DB_TMP_DIR}/graph0.yaml" - # add some default settings and non-user-aware settings to schema file. - amplify_graph_schema ${schema_file} ${amplify_schema_file} - # check graph is running inside docker - check_graph_not_running ${graph_name} || (err "Can not create graph ${graph_name}, since a graph with same naming running." && exit 1) - # create the graph directory in the docker's workspace - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - docker_graph_schema_file="${docker_graph_dir}/graph.yaml" - docker_graph_schema_file_back="${docker_graph_dir}/graph0.yaml" # used for later adding/removing stored procedure - # check docker_graph_schema_file exists in the container, if exists, tell user to remove it first - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_schema_file} ] && echo -e \"${RED} Graph ${graph_name} already exists, please remove it first. ${NC}\" && exit 1 || exit 0" || exit 1 - # create the graph directory in the docker's workspace - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${docker_graph_dir}" || exit 1 - # create plugins dir - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "mkdir -p ${docker_graph_dir}/plugins" || exit 1 - # copy the schema file to the docker's workspace - docker cp "${amplify_schema_file}" "${GIE_DB_CONTAINER_NAME}:${docker_graph_schema_file}" || exit 1 - docker cp "${amplify_schema_file}" "${GIE_DB_CONTAINER_NAME}:${docker_graph_schema_file_back}" || exit 1 - info "Successfully create graph [${graph_name}]" - #TODO: support creating an empty graph -} - -###################Remove graph#################### -function do_remove(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift # past argument - shift - ;; - *) - err "unknown option $1" - create_usage - exit 1 - ;; - esac - done - # check graph_name is set - if [ -z "${graph_name}" ]; then - err "graph name is not specified" - remove_usage - exit 1 - fi - check_graph_name_valid "${graph_name}" - # check if the graph is created - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be removed, since it is not created." - exit 1 - fi - # check graph is running inside docker - if ! check_graph_not_running ${graph_name}; then - err "Can not remove graph ${graph_name}, since a graph with same naming running!" - exit 1; - fi - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - ### let user confirm to remove the graph - read -p "Are you sure to remove graph [${graph_name}?] [y/n]" -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - info "Abort removing graph [${graph_name}]" - exit 1 - fi - - # rm -rf the graph directory in the docker's workspace - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -rf ${docker_graph_dir}" || ( echo "Fail to delete graph ${graph_name}" && exit 1) - info "Successfully remove graph [${graph_name}]" - # if graph_name is current graph, set current graph to .running - if [ "${DATABASE_CURRENT_GRAPH_NAME}" = "${graph_name}" ]; then - info "Remove current graph [${graph_name}], set current graph to default_graph: ${DATABASE_DEFAULT_GRAPH_NAME}" - echo "export DATABASE_CURRENT_GRAPH_NAME=${DATABASE_DEFAULT_GRAPH_NAME}" >> ${HOST_DB_ENV_FILE} - fi - #local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - # if the removed graph is in .running, remove it - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - if [ "${graph_name}" = "${graph_name}" ]; then - rm ${HOST_DB_RUNNING_FILE} - fi - fi -} - -#################### Import #################### -function do_import(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift # past argument - shift - ;; - -c | --config) - bulk_load_file="$2" - shift - shift - ;; - *) - err "unknown option $1" - import_usage - exit 1 - ;; - esac - done - # check graph_name is set - if [ -z "${graph_name}" ]; then - err "graph name is not specified" - import_usage - exit 1 - fi - check_graph_name_valid "${graph_name}" - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be found, please create it first" - exit 1 - fi - info "Import data to graph [${graph_name}] from ${bulk_load_file}" - # check if the container is running - check_container_running - # check if the bulk_load_file exists - check_file_exists "${bulk_load_file}" - info "bulk_load_file ${bulk_load_file} exists" - - check_graph_not_running ${graph_name} || info "Can not import data to graph [${graph_name}], since it is already running." - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - # check graph_schema_file exists in container, if not, let user create graph first - docker_graph_schema_file="${DATABASE_WORKSPACE}/data/${graph_name}/graph.yaml" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_schema_file} ] || (echo -e \"${RED}graph [${graph_name}] not exists, please create it first.${NC}\" && exit 1)" - info "Graph Schema exists" - # copy the bulk_load_file to container - bulk_load_file_name=$(basename "${bulk_load_file}") - docker_bulk_load_file="${HOST_DB_TMP_DIR}/${bulk_load_file_name}" - docker cp "${bulk_load_file}" "${GIE_DB_CONTAINER_NAME}:${docker_bulk_load_file}" - - docker_graph_data_dir="${DATABASE_WORKSPACE}/data//${graph_name}/indices" - # currently we can only overwrite the indices, so if it exists, remove it first - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_data_dir} ] && rm -rf ${docker_graph_data_dir} || exit 0" || exit 1 - - cmd="${DOCKER_DB_GRAPH_IMPORT_BIN} -g ${docker_graph_schema_file} -l ${docker_bulk_load_file} -d ${docker_graph_data_dir}" - info "Running cmd: ${cmd}" - # docker exec the cmd, fail if cmd fail - if ! docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "${cmd} ||(echo -e \"${RED} Fail to import graph [${graph_name}]${NC}\" && exit 1 )"; then - err "Fail to import graph [${graph_name}]" - exit 1 - fi - info "Successfully import data to graph [${graph_name}]" -} - -#################### List Graph #################### -function do_list() { - # check if the container is running - check_container_running - # get all graph names - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - host_data_dir="${HOST_DB_HOME}/${DATABASE_DATA_DIR_NAME}" - # get all graph names into a array - graph_names=($(ls ${host_data_dir})) - # if graph_names is empty, no graph exists - if [ ${#graph_names[@]} -eq 0 ]; then - info "No graph has been created." - exit 0 - fi - # print all graph names - for graph_name in "${graph_names[@]}"; do - echo "Found graph: [${graph_name}]" - done - -} - -#################### Destroy #################### -function do_destroy() { - - # if container not exists, exit - if ! docker ps -a --format '{{.Names}}' | grep -Eq "^${GIE_DB_CONTAINER_NAME}$"; then - info "Database has not been created, exit" - exit 0 - fi - # check if the GraphServer and SyncServer are running, if running, echo "Service is running, please stop it first" and exit - if ! check_process_not_running_in_container ${GIE_DB_CONTAINER_NAME} "GraphServer" "Service is running, please stop it first: ./bin/gs_interactive service stop"; then - exit 1 - fi - if ! check_process_not_running_in_container ${GIE_DB_CONTAINER_NAME} "interactive_server" "Service is running, please stop it first: ./bin/gs_interactive service stop"; then - exit 1 - fi - - # let user confirm to destroy the database - read -p "Are you sure to destroy the database? [y/n]" -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - info "Abort destroying database" - exit 1 - fi - - info "Destroying database..." - docker stop "${GIE_DB_CONTAINER_NAME}" - docker rm "${GIE_DB_CONTAINER_NAME}" - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - # rm host data/* - sudo rm -rf ${HOST_DB_HOME}/data/* - - #rm .running - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - rm ${HOST_DB_RUNNING_FILE} - fi - if [ -f "${HOST_DB_ENV_FILE}" ]; then - rm ${HOST_DB_ENV_FILE} - fi - # rm the temp files used - if [ -f "${HOST_DB_TMP_DIR}/graph0.yaml" ]; then - rm ${HOST_DB_TMP_DIR}/graph0.yaml - fi - if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then - rm ${HOST_DB_TMP_DIR}/.enable - fi - # rm ${HOST_DB_TMP_DIR}/engine_config.yaml ${HOST_DB_TMP_DIR}/real_engine_config.yaml, ${HOST_DB_TMP_DIR}/graph_exists ${HOST_DB_TMP_DIR}/graph_loaded - if [ -f "${HOST_DB_TMP_DIR}/engine_config.yaml" ]; then - rm ${HOST_DB_TMP_DIR}/engine_config.yaml - fi - if [ -f "${HOST_DB_TMP_DIR}/real_engine_config.yaml" ]; then - rm ${HOST_DB_TMP_DIR}/real_engine_config.yaml - fi - if [ -f "${HOST_DB_TMP_DIR}/graph_exists" ]; then - rm ${HOST_DB_TMP_DIR}/graph_exists - fi - if [ -f "${HOST_DB_TMP_DIR}/graph_loaded" ]; then - rm ${HOST_DB_TMP_DIR}/graph_loaded - fi - - info "Finish destroy database" -} - -#################### Start database #################### -function do_start(){ - check_database_initialized - if [ -f "${HOST_DB_ENV_FILE}" ];then - . ${HOST_DB_ENV_FILE} - fi - info "Starting database..." - - # check whether the .running file exists, if exists, exit - local compiler_process_id=$(docker top "${GIE_DB_CONTAINER_NAME}" | grep "${DOCKER_DB_COMPILER_BIN}" | awk '{print $2}') - local server_process_id=$(docker top "${GIE_DB_CONTAINER_NAME}" | grep "${DOCKER_DB_SERVER_BIN}" | awk '{print $2}') - # if both process are running, exit - # if only one process is running, kill the process - if [ -n "${compiler_process_id}" ] && [ -n "${server_process_id}" ]; then - local old_graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - err "Database is already running on [${old_graph_name}], please stop it first" - exit 1 - fi - if [ -n "${compiler_process_id}" ]; then - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "kill -9 ${compiler_process_id}" - fi - if [ -n "${server_process_id}" ]; then - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "kill -9 ${server_process_id}" - fi - - - # set trap to do_stop - trap do_stop SIGINT SIGTERM - - graph_name="" - engine_config_file="" - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift # past argument - shift - ;; - -c | --config) - engine_config_file="$2" - shift - shift - ;; - *) - err "unknown option $1" - start_usage - exit 1 - ;; - esac - done - # try parse default_graph from engine_config_file - # generate real engine config file, put it at ${HOST_DB_TMP_DIR}/real_engine_config.yaml - if [ -z "${graph_name}" ]; then - graph_name=${DATABASE_CURRENT_GRAPH_NAME} - else - DATABASE_CURRENT_GRAPH_NAME=${graph_name} - info "Using user specified graph [${graph_name}]" - fi - check_graph_name_valid "${graph_name}" - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be found, please create it and import data first." - exit 1 - fi - - real_engine_config_file="${HOST_DB_TMP_DIR}/real_engine_config.yaml" - if [ -z "${engine_config_file}" ]; then - if ! generate_real_engine_conf "${real_engine_config_file}"; then - err "generate engine config file failed" - exit 1 - fi - else - if ! generate_real_engine_conf "${engine_config_file}" "${real_engine_config_file}"; then - err "generate engine config file failed" - exit 1 - fi - fi - - # copy engine config file to container - dst_engine_config_file="${DATABASE_WORKSPACE}/conf/engine_config.yaml" - docker cp "${real_engine_config_file}" "${GIE_DB_CONTAINER_NAME}:${dst_engine_config_file}" || (echo "fail to copy $engine_config_file to container" && exit 1) - - - # check if modern_graph exists in container, get the result as bool - docker_graph_schema_file="${DATABASE_WORKSPACE}/data/${graph_name}/graph.yaml" - flag_file="${DATABASE_WORKSPACE}/data/${graph_name}/indices/schema" - if [ -f "${HOST_DB_TMP_DIR}/graph_exists" ]; then - if ! rm "${HOST_DB_TMP_DIR}/graph_exists"; then - err "fail to remove ${HOST_DB_TMP_DIR}/graph_exists, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/graph_exists" - exit 1 - fi - fi - if [ -f "${HOST_DB_TMP_DIR}/graph_loaded" ]; then - if ! rm "${HOST_DB_TMP_DIR}/graph_loaded"; then - err "fail to remove ${HOST_DB_TMP_DIR}/graph_loaded, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/graph_loaded" - exit 1 - fi - fi - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${docker_graph_schema_file} ] && echo \"true\" e) || echo \"false\"" > ${HOST_DB_TMP_DIR}/graph_exists - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "( [ -f ${flag_file} ] && echo \"true\" e) || echo \"false\"" > ${HOST_DB_TMP_DIR}/graph_loaded - graph_exists=$(cat ${HOST_DB_TMP_DIR}/graph_exists) - graph_loaded=$(cat ${HOST_DB_TMP_DIR}/graph_loaded) - if [ "${graph_exists}" = "false" ]; then - # if graph_name is default_graph, we should create it first - # otherwise, we should tell user to create it first - if [ "${graph_name}" != "${DATABASE_DEFAULT_GRAPH_NAME}" ]; then - err "graph [${graph_name}] not exists, please create it first" - exit 1 - fi - fi - - if [ "${graph_loaded}" = "false" ]; then - err "graph [${graph_name}] is empty, please import data first" - exit 1 - fi - - info "graph [${graph_name}] exists, start it" - - do_stop - ensure_container_running - # regenerate graph.yaml from graph0.yaml and override graph.yaml with stored procedure enable and disable - update_graph_yaml_with_procedure_enabling ${graph_name} - - # the bulk_load_file should place inside ${DATABASE_WORKSPACE}. and should use relative path - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - info "In start database, received graph_name = ${graph_name}, engine_config_file = ${engine_config_file}" - docker_server_log_path="${DATABASE_WORKSPACE}/logs/server.log" - graph_schema_file="${DATABASE_WORKSPACE}/data/${graph_name}/graph.yaml" - csr_data_dir="${DATABASE_WORKSPACE}/data/${graph_name}/indices" - cmd="docker exec ${GIE_DB_CONTAINER_NAME} bash -c \"" - cmd="${cmd} ${DOCKER_DB_SERVER_BIN} -c ${dst_engine_config_file}" - cmd="${cmd} -g ${graph_schema_file} --data-path ${csr_data_dir}" - cmd="${cmd} > ${docker_server_log_path} 2>&1 & \"" - echo "Running cmd: ${cmd}" - # eval command, if fails exist - eval ${cmd} || (echo "Fail to launch hqps server" && exit 1) - sleep 4 - # check whether the process is running - check_process_running_in_container ${GIE_DB_CONTAINER_NAME} ${DOCKER_DB_SERVER_BIN} ", use gs_interactive service get_log -o [dir] to see get logs" - info "Successfully start server" - - # start compiler - docker_compiler_log_path="${DATABASE_WORKSPACE}/logs/compiler.log" - cmd="docker exec ${GIE_DB_CONTAINER_NAME} bash -c \"" - cmd=${cmd}"java -cp \"${DOCKER_DB_GIE_HOME}/lib/*:${DOCKER_DB_GIE_HOME}/lib/compiler-0.0.1-SNAPSHOT.jar\" " - cmd=${cmd}" -Djna.library.path=${DOCKER_DB_GIE_HOME}/lib/" - cmd=${cmd}" -Dgraph.schema=${graph_schema_file}" - # should error be reported? - # cmd=${cmd}" -Dgraph.stored.procedures.uri=file:${docker_graph_plugin_dir}" - cmd=${cmd}" ${DOCKER_DB_COMPILER_BIN} ${dst_engine_config_file} > ${docker_compiler_log_path} 2>&1 &" - cmd=${cmd}"\"" - info "Running cmd: ${cmd}" - eval ${cmd} - sleep 6 - check_process_running_in_container ${GIE_DB_CONTAINER_NAME} ${DOCKER_DB_COMPILER_BIN} ", use gs_interactive service get_log -o [dir] to see more details" - info "Successfully start compiler" - # get cypher port from engine config file - eval $(parse_yaml "${real_engine_config_file}") - - if [ "${compiler_endpoint_bolt_connector_disabled}" = "false" ]; then - emph "Cypher port is open on :${compiler_endpoint_bolt_connector_port}" - else - emph "Bolt connector is disabled, no bolt endpoint is available" - fi - if [ "${compiler_endpoint_gremlin_connector_disabled}" = "false" ]; then - emph "Gremlin port is running, port is open on :${compiler_endpoint_gremlin_connector_port}" - else - emph "Gremlin connector is disabled, no gremlin endpoint is available" - fi - - # if do_start success, we should write current args to ${HOST_DB_RUNNING_FILE} - echo "GRAPH_NAME=${graph_name}" > ${HOST_DB_RUNNING_FILE} - echo "ENGINE_CONFIG_FILE=${engine_config_file}" >> ${HOST_DB_RUNNING_FILE} - # create .lock file - docker_graph_lock_file="${DATABASE_WORKSPACE}/data/${graph_name}/.lock" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "touch ${docker_graph_lock_file}" || exit 1 -} - - -#################### Stop database #################### -function do_stop(){ - # if container is not running, do nothing - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - else - info "No running database found, do nothing" - fi - # get graph_name from ${HOST_DB_RUNNING_FILE} - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - docker_graph_lock_file="${DATABASE_WORKSPACE}/data/${graph_name}/.lock" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "rm -f ${docker_graph_lock_file}" || exit 1 - info "Successfully remove ${docker_graph_lock_file} file" - else - info "No graph is running" - fi - - # stop the SERVER_BIN process and graph_server process - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "pkill -f ${DOCKER_DB_SERVER_BIN}" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "pkill -f ${DOCKER_DB_COMPILER_BIN}" - sleep 6 - info "Successfully stop database" -} - - -#################### Get database status #################### -function do_status() { - if [ "$(docker inspect -f '{{.State.Running}}' "${GIE_DB_CONTAINER_NAME}")" = "true" ]; then - info "container ${GIE_DB_CONTAINER_NAME} is running" - else - info "container ${GIE_DB_CONTAINER_NAME} is not running" - info "Please start database first" - fi - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - # the container is running but the process is not running - check_process_running_in_container ${GIE_DB_CONTAINER_NAME} ${DOCKER_DB_SERVER_BIN} "The service is stopped or down. Use gs_interactive service get_log -o [dir] to see more details" - check_process_running_in_container ${GIE_DB_CONTAINER_NAME} ${DOCKER_DB_COMPILER_BIN} "The service is stopped or down. Use gs_interactive service get_log -o [dir] to see more details" - # get cypher port from engine config file in container - - docker_engine_config_file="${DATABASE_WORKSPACE}/conf/engine_config.yaml" - # copy the engine config file to host's tmp directory - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_engine_config_file}" "${HOST_DB_TMP_DIR}/engine_config.yaml" || exit 1 - eval $(parse_yaml "${HOST_DB_TMP_DIR}/engine_config.yaml") - if [ "${compiler_endpoint_bolt_connector_disabled}" = "false" ]; then - emph "Cypher port is open on :${compiler_endpoint_bolt_connector_port}" - else - emph "Bolt connector is disabled, no bolt endpoint is available" - fi - if [ "${compiler_endpoint_gremlin_connector_disabled}" = "false" ]; then - emph "Gremlin port is open on :${compiler_endpoint_gremlin_connector_port}" - else - emph "Gremlin connector is disabled, no gremlin endpoint is available" - fi - # print current running graph name - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - emph "Current running graph is [${graph_name}]" - else - err "No graph is running" - fi -} - - - -#################### Restart #################### -function do_restart() { - # read args from cached file. - # get num lines in file ${HOST_DB_RUNNING_FILE} - num_lines=$(wc -l < ${HOST_DB_RUNNING_FILE}) - if [ ${num_lines} -ne 2 ]; then - err "Error: ${HOST_DB_RUNNING_FILE} should have 2 lines, but got ${num_lines}, something wrong with the file ${HOST_DB_RUNNING_FILE}" - exit 1 - fi - # read args from file - GRAPH_NAME=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - ENGINE_CONFIG_FILE=$(sed -n '2p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2) - # parse current args, override the args from file - info "Restarting database..." - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - GRAPH_NAME="$2" - shift # past argument - shift - ;; - -c | --config) - ENGINE_CONFIG_FILE="$2" - shift - shift - ;; - *) - err "unknown option $1" - restart_usage - exit 1 - ;; - esac - done - do_stop - info "Successfully stop database" - do_start -g ${GRAPH_NAME} -c ${ENGINE_CONFIG_FILE} - info "Finish restarting database..." -} - -#################### Get log #################### -function do_log(){ - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -o | --output) - directory="$2" - shift # past argument - shift - ;; - *) - err "unknown option $1" - get_log_usage - exit 1 - ;; - esac - done - # check directory is set - if [ -z "${directory}" ]; then - err "output directory is not specified" - get_log_usage - exit 1 - fi - # get log directory in container - docker_log_dir="${DATABASE_WORKSPACE}/logs" - # copy ${docker_log_dir}/compiler.log and ${docker_log_dir}/server.log to ${directory} - docker_compiler_log="${docker_log_dir}/compiler.log" - docker_server_log="${docker_log_dir}/server.log" - # docker cp - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_compiler_log}" "${directory}/compiler.log" || exit 1 - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_server_log}" "${directory}/server.log" || exit 1 - info "Successfully get log to ${directory}, please check compiler.log and server.log" -} - -# the compiled dynamic libs will be placed at data/${graph_name}/plugins/ -# after compilation, the user need to write the corresponding yaml, telling the compiler about -# the input and output of the stored procedure -function do_compile() { - ensure_container_running - if [ $# -lt 2 ]; then - err "compile stored_procedure command at least 2 args, but got $#" - compile_usage - exit 1 - fi - compile_only=false - - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - info "graph_name = ${graph_name}" - shift # past argument - shift - ;; - -i | --input) - file_path="$2" - shift # past argument - shift - ;; - -n | --name) - stored_procedure_name="$2" - shift - shift - ;; - -d | --description) - stored_procedure_desc="$2" - shift - shift - ;; - --compile_only) - compile_only=true - shift - shift - ;; - *) - err "unknown option $1" - compile_usage - exit 1 - ;; - esac - done - - # check graph_name - if [ -z "${graph_name}" ]; then - # let user confirm that no graph_name is specified, will use default graph. - read -p "No graph_name is specified, will use default graph, continue? [y/n]" -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - info "Abort compile stored_procedure" - compile_usage - exit 1 - fi - info "Using default graph [${DATABASE_DEFAULT_GRAPH_NAME}]" - graph_name=${DATABASE_DEFAULT_GRAPH_NAME} - fi - check_graph_name_valid "${graph_name}" - # check graph exists - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be found, please create it first" - exit 1 - fi - - # check file_path - check_file_exists "${file_path}" - # get real file_path - file_name=$(basename "${file_path}") - # get file_name and assign to stored_procedure_name if stored_procedure_name is not set - if [ -z "${stored_procedure_name}" ]; then - stored_procedure_name="${file_name%.*}" - fi - real_file_path=$(realpath "${file_path}") - # check exists - if [ ! -f "${real_file_path}" ]; then - err "file ${real_file_path} not exist" - exit 1 - fi - - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - - real_engine_config_file="${HOST_DB_TMP_DIR}/real_engine_config.yaml" - # update default graph name - DATABASE_CURRENT_GRAPH_NAME=${graph_name} - if ! generate_real_engine_conf "${real_engine_config_file}"; then - err "Fail to generate real engine config file, please remove ${real_engine_config_file} and try again" - exit 1 - fi - # copy to container - docker_engine_config="${DATABASE_WORKSPACE}/conf/engine_config.yaml" - docker cp "${real_engine_config_file}" "${GIE_DB_CONTAINER_NAME}:${docker_engine_config}" || exit 1 - - docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - docker_graph_schema="${docker_graph_dir}/graph.yaml" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo -e \"${RED} Graph [${graph_name}] not exists, please create it first.${NC}\" && exit 1)" - # Fetch current installed procedures, and check if the procedure is already installed - # if not compile_only, we should add the stored_procedure_name to .enable - docker_graph_enable_file="${docker_graph_dir}/plugins/.enable" - # copy container to host - if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then - if ! rm -f ${HOST_DB_TMP_DIR}/.enable; then - err "fail to remove ${HOST_DB_TMP_DIR}/.enable, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/.enable" - exit 1 - fi - fi - # if docker_graph_enable_file exists. copy it to host - docker exec "${GIE_DB_CONTAINER_NAME}" test -e "${docker_graph_enable_file}" && (docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable") - - if [ ! -f "${HOST_DB_TMP_DIR}/.enable" ]; then - touch "${HOST_DB_TMP_DIR}/.enable" - fi - # check if the stored_procedure_name is already in .enable, exact match - if grep -q "^${stored_procedure_name}$" "${HOST_DB_TMP_DIR}/.enable"; then - err "stored_procedure_name [${stored_procedure_name}] already exists, please use another name" - exit 1 - fi - - container_output_dir="${DATABASE_WORKSPACE}/data/${graph_name}/plugins" - container_input_path="${HOST_DB_TMP_DIR}/${file_name}" - # docker cp file to container - cmd="docker cp ${real_file_path} ${GIE_DB_CONTAINER_NAME}:${container_input_path}" - eval ${cmd} || exit 1 - - cmd="docker exec ${GIE_DB_CONTAINER_NAME} bash -c '" - cmd=${cmd}" ${DOCKER_DB_GEN_BIN}" - cmd=${cmd}" --engine_type=hqps" - cmd=${cmd}" --input=${container_input_path}" - cmd=${cmd}" --work_dir=/tmp/codegen/" - cmd=${cmd}" --ir_conf=${docker_engine_config}" - cmd=${cmd}" --graph_schema_path=${docker_graph_schema}" - cmd=${cmd}" --output_dir=${container_output_dir}" - cmd=${cmd}" --procedure_name=${stored_procedure_name}" - if [ ! -z "${stored_procedure_desc}" ]; then - cmd=${cmd}" --procedure_desc=\"${stored_procedure_desc}\"" - fi - cmd=${cmd}" '" - - echo "Running cmd: ${cmd}" - eval ${cmd} || exit 1 - # check output exists - # remove the suffix of file_name - output_file="${HOST_DB_HOME}/data/${graph_name}/plugins/lib${stored_procedure_name}.so" - - if [ ! -f "${output_file}" ]; then - err "output file ${output_file} not exist, compilation failed" - exit 1 - fi - info "success generate dynamic lib ${output_file}." - - # if compile_only equal to false - if [ "${compile_only}" = false ]; then - echo "${stored_procedure_name}" >> ${HOST_DB_TMP_DIR}/.enable - fi - # copy back - docker cp "${HOST_DB_TMP_DIR}/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 -} - -function do_enable(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift # past argument - shift - ;; - -n | --name) - stored_procedure_names="$2" - shift - shift - ;; - -c | --config) - stored_procedure_names_yaml="$2" - shift - shift - ;; - *) - err "unknown option $1" - enable_proc_usage - exit 1 - ;; - esac - done - # check graph_name not empty - # check graph_name - if [ -z "${graph_name}" ]; then - # let user confirm that no graph_name is specified, will use default graph. - read -p "No graph_name is specified, will use default graph, continue? [y/n]" -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - info "Abort compile stored_procedure" - enable_proc_usage - exit 1 - fi - info "Using default graph [${DATABASE_DEFAULT_GRAPH_NAME}]" - graph_name=${DATABASE_DEFAULT_GRAPH_NAME} - fi - check_graph_name_valid "${graph_name}" - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be found, please create it first" - exit 1 - fi - - # --name and --config can not be set at the same time - if [ ! -z "${stored_procedure_names}" ] && [ ! -z "${stored_procedure_names_yaml}" ]; then - err "--name and --config can not be set at the same time" - enable_proc_usage - exit 1 - fi - # use stored_procedures_names_yaml if it is set - if [ ! -z "${stored_procedure_names_yaml}" ]; then - check_file_exists "${stored_procedure_names_yaml}" - # cat the file and get each line as a stored_procedure_name, join them with ',' - stored_procedure_names=$(< ${stored_procedure_names_yaml} tr '\n' ',' | sed 's/,$//') - fi - info "stored_procedure_names = ${stored_procedure_names}" - # add the names to .enable file for graph_name - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo -e \"${RED} Graph ${graph_name} not exists, please create it first.${NC}\" && exit 1)" - docker_graph_plugin_dir="${docker_graph_dir}/plugins" - docker_graph_enable_file="${docker_graph_plugin_dir}/.enable" - if [ -f "${HOST_DB_TMP_DIR}/.enable" ]; then - if ! rm -f ${HOST_DB_TMP_DIR}/.enable; then - err "fail to remove ${HOST_DB_TMP_DIR}/.enable, please remove it manually, maybe sudo rm -rf ${HOST_DB_TMP_DIR}/.enable" - exit 1 - fi - fi - # copy the .enable file to host, and append the stored_procedure_names to it; if the stored_procedure_names already exists, do nothing - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable" || true - if [ ! -f "${HOST_DB_TMP_DIR}/.enable" ]; then - touch "${HOST_DB_TMP_DIR}/.enable" - fi - old_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) - # split the stored_procedure_names by ',' and append them to .enable file - IFS=',' read -ra stored_procedure_names_array <<< "${stored_procedure_names}" - for stored_procedure_name in "${stored_procedure_names_array[@]}"; do - # check if the stored_procedure_name already exists in .enable file - if grep -q "${stored_procedure_name}" "${HOST_DB_TMP_DIR}/.enable"; then - info "stored_procedure_name ${stored_procedure_name} already exists in .enable file, skip" - else - echo "${stored_procedure_name}" >> ${HOST_DB_TMP_DIR}/.enable - fi - done - # copy the .enable file back to container - docker cp "${HOST_DB_TMP_DIR}/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 - new_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) - info "Successfully enable stored_procedures ${stored_procedure_names} for graph [${graph_name}], ${old_line_num} -> ${new_line_num}" -} - -function do_disable(){ - disable_all=false - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift # past argument - shift - ;; - -n | --name) - stored_procedure_names="$2" - shift - shift - ;; - -c | --config) - stored_procedure_names_yaml="$2" - shift - shift - ;; - -a | --all) - disable_all=true - shift - ;; - *) - err "unknown option $1" - disable_proc_usage - exit 1 - ;; - esac - done - - # --name and --config can not be set at the same time - if [ ! -z "${stored_procedure_names}" ] && [ ! -z "${stored_procedure_names_yaml}" ] && [ "${disable_all}" = true ]; then - err "--name, --config and --all can not be set at the same time" - disable_proc_usage - exit 1 - fi - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - # check graph_name not empty - if [ -z "${graph_name}" ]; then - # let user confirm that no graph_name is specified, will use default graph. - read -p "No graph_name is specified, will use default graph, continue? [y/n]" -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - info "Abort compile stored_procedure" - disable_proc_usage - exit 1 - fi - info "Using default graph [${DATABASE_DEFAULT_GRAPH_NAME}]" - graph_name=${DATABASE_DEFAULT_GRAPH_NAME} - fi - check_graph_name_valid "${graph_name}" - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be found, please create it first" - exit 1 - fi - - info "graph_name = ${graph_name}" - docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo \"graph ${graph_name} not exists, please create it first\" && exit 1)" - docker_graph_plugin_dir="${docker_graph_dir}/plugins" - docker_graph_enable_file="${docker_graph_plugin_dir}/.enable" - echo "disable_all = ${disable_all}" - if [ "${disable_all}" = true ]; then - # clear the .enable file - info "disable all stored_procedures for graph [${graph_name}]" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "(rm \"\" > ${docker_graph_enable_file}) || exit 0" || exit 1 - info "Successfully disable all stored_procedures for graph [${graph_name}]" - exit 0 - fi - - # use stored_procedures_names_yaml if it is set - if [ ! -z "${stored_procedure_names_yaml}" ]; then - check_file_exists "${stored_procedure_names_yaml}" - # cat the file and get each line as a stored_procedure_name, join them with ',' - stored_procedure_names=$(< ${stored_procedure_names_yaml} tr '\n' ',' | sed 's/,$//') - fi - info "stored_procedure_names = ${stored_procedure_names}" - # add the names to .enable file for graph_name - - # copy the .enable file to host, and remove the stored_procedure_names from it - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable" || exit 1 - old_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) - # split the stored_procedure_names by ',' and remove them from .enable file - IFS=',' read -ra stored_procedure_names_array <<< "${stored_procedure_names}" - for stored_procedure_name in "${stored_procedure_names_array[@]}"; do - sed -i "/${stored_procedure_name}/d" ${HOST_DB_TMP_DIR}/.enable - done - # copy the .enable file back to container - docker cp "${HOST_DB_TMP_DIR}/.enable" "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" || exit 1 - new_line_num=$(wc -l < ${HOST_DB_TMP_DIR}/.enable) - info "Successfully disable stored_procedures ${stored_procedure_names} for graph [${graph_name}], ${old_line_num} -> ${new_line_num}" -} - -function do_show(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -g | --graph) - graph_name="$2" - shift - shift - ;; - *) - err "unknown option $1" - show_stored_procedure_usage - exit 1 - ;; - esac - done - if [ -f "${HOST_DB_RUNNING_FILE}" ]; then - . ${HOST_DB_ENV_FILE} - fi - # check graph_name - if [ -z "${graph_name}" ]; then - # let user confirm that no graph_name is specified, will use default graph. - read -p "No graph_name is specified, will use default graph, continue? [y/n]" -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - info "Abort compile stored_procedure" - show_stored_procedure_usage - exit 1 - fi - info "Using default graph [${DATABASE_DEFAULT_GRAPH_NAME}]" - graph_name=${DATABASE_DEFAULT_GRAPH_NAME} - fi - check_graph_name_valid "${graph_name}" - if [ ! -d "${HOST_DB_HOME}/data/${graph_name}" ]; then - err "graph [${graph_name}] can not be found, please create it first" - exit 1 - fi - info "graph_name = ${graph_name}" - docker_graph_dir="${DATABASE_WORKSPACE}/data/${graph_name}" - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -d ${docker_graph_dir} ] || (echo -e \"${RED}Graph [${graph_name}] not exists, please create it first. ${NC}\" && exit 1)" || exit 1 - docker_graph_yaml="${docker_graph_dir}/graph.yaml" - # check if docker_graph_yaml exists, if not ,exit - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_yaml} ] || (echo -e \"${RED}Graph [${graph_name}] not exists, please create it first. ${NC}\" && exit 1)" || exit 1 - # copy to host - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_yaml}" "${HOST_DB_TMP_DIR}/graph.yaml" || (echo "fail to copy" && exit 1) - # parse /tmp/graph.yaml and get stored_procedures_enable_lists array - eval $(parse_yaml "${HOST_DB_TMP_DIR}/graph.yaml") - x=1 - stored_procedures_enable_lists_array=() - while true; do - procedure_name_key="stored_procedures_enable_lists_${x}" - procedure_name_value_x=$(eval echo "\$${procedure_name_key}") - if [ -z "${procedure_name_value_x}" ]; then - break - fi - #append value to array - stored_procedures_enable_lists_array+=("${procedure_name_value_x}") - x=$((x + 1)) - done - info "Enabled Size: ${#stored_procedures_enable_lists_array[@]}" - - # parse the .enabled file - docker_graph_plugin_dir="${docker_graph_dir}/plugins" - docker_graph_enable_file="${docker_graph_plugin_dir}/.enable" - # check if docker_graph_enable_file exists, if not ,exit - docker exec "${GIE_DB_CONTAINER_NAME}" bash -c "[ -f ${docker_graph_enable_file} ] || (echo -e \"${RED}Graph [${graph_name}] has no procedures registered. ${NC}\" && exit 1)" || exit 1 - docker cp "${GIE_DB_CONTAINER_NAME}:${docker_graph_enable_file}" "${HOST_DB_TMP_DIR}/.enable" || exit 1 - disabled_list=() - # iterate the .enable file, for each line, check if it is in stored_procedures_enable_lists_array, if not, add it to disabled_list - while read line; do - if [[ ! " ${stored_procedures_enable_lists_array[@]} " =~ " ${line} " ]]; then - disabled_list+=("${line}") - fi - done < ${HOST_DB_TMP_DIR}/.enable - info "Disabled Size: ${#disabled_list[@]}" - - # print the enabled_list and disabled_list - for procedure_name in "${stored_procedures_enable_lists_array[@]}"; do - emph " ${procedure_name} : [Enabled]" - done - for procedure_name in "${disabled_list[@]}"; do - emph " ${procedure_name} : [Disabled]" - done -} - -function do_database(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - create) - shift - do_create "$@" - exit 0 - ;; - remove) - shift - do_remove "$@" - exit 0 - ;; - import) - shift - do_import "$@" - exit 0 - ;; - list) - shift - do_list "$@" - exit 0 - ;; - *) - err "unknown option $1" - database_usage - exit 1 - ;; - esac - done -} - -function do_service(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - start) - shift - do_start "$@" - exit 0 - ;; - stop) - shift - do_stop "$@" - exit 0 - ;; - restart) - shift - do_restart "$@" - exit 0 - ;; - status) - shift - do_status "$@" - exit 0 - ;; - get_log) - shift - do_log "$@" - exit 0 - ;; - *) - err "unknown option $1" - services_usage - exit 1 - ;; - esac - done -} - -function do_procedure(){ - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - compile) - shift - do_compile "$@" - exit 0 - ;; - enable) - shift - do_enable "$@" - exit 0 - ;; - disable) - shift - do_disable "$@" - exit 0 - ;; - show) - shift - do_show "$@" - exit 1 - ;; - *) - err "unknown option $1" - procedure_usage - exit 1 - ;; - esac - done - procedure_usage -} - -#################### Entry #################### -if [ $# -eq 0 ]; then - usage - exit 1 -fi - -while [[ $# -gt 0 ]]; do - key="$1" - - case $key in - -h | --help) - usage - exit - ;; - init) - shift - info "Start initiating database..." - do_init "$@" - exit 0 - ;; - database) - shift - check_database_initialized - do_database "$@" - exit 0 - ;; - service) - shift - check_database_initialized - do_service "$@" - exit 0 - ;; - procedure) - shift - check_database_initialized - do_procedure "$@" - exit 0 - ;; - destroy) - shift - do_destroy "$@" - exit 0 - ;; - *) # unknown option - err "unknown option $1" - usage - exit 1 - ;; - esac -done \ No newline at end of file diff --git a/flex/interactive/conf/engine_config.yaml b/flex/interactive/conf/engine_config.yaml deleted file mode 100644 index a11b88fb01b0..000000000000 --- a/flex/interactive/conf/engine_config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -log_level: INFO # default INFO -compute_engine: - thread_num_per_worker: 1 # the number of shared workers, default 1 -compiler: - planner: - is_on: true - opt: RBO - rules: - - FilterIntoJoinRule - - FilterMatchRule - - NotMatchToAntiJoinRule - query_timeout: 20000 # query timeout in milliseconds, default 2000 \ No newline at end of file diff --git a/flex/interactive/conf/interactive.yaml b/flex/interactive/conf/interactive.yaml deleted file mode 100755 index 6897acaf5363..000000000000 --- a/flex/interactive/conf/interactive.yaml +++ /dev/null @@ -1,3 +0,0 @@ -version: v0.3 -#volume: -# - {HOST_PATH}:${CONTAINER_PATH} \ No newline at end of file diff --git a/k8s/Makefile b/k8s/Makefile index e58d557fee82..14b091feba07 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -155,7 +155,7 @@ interactive-experimental: flex-interactive: cd $(WORKING_DIR)/.. && \ docker build \ - -f ${WORKING_DIR}/flex-interactive.Dockerfile \ + -f ${DOCKERFILES_DIR}/flex-interactive.Dockerfile \ --target runtime \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index ce3eca42509d..641a16539020 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -6,6 +6,10 @@ ARG ARCH=x86_64 ARG ENABLE_COORDINATOR="false" FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +RUN sudo mkdir -p /opt/flex && sudo chown -R graphscope:graphscope /opt/flex/ +USER graphscope +WORKDIR /home/graphscope + # change bash as default SHELL ["/bin/bash", "-c"] @@ -87,6 +91,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # copy builder's /opt/flex to final image COPY --from=builder /opt/flex /opt/flex +COPY --from=builder /opt/graphscope/lib/libgrape-lite.so /opt/flex/lib/ # copy the builtin graph, modern_graph RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ From 1470925bab7cdcf9fb591f717a78b30ae9bbc499 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 6 Aug 2024 20:08:20 +0800 Subject: [PATCH 04/10] fix format Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container --- python/graphscope/gsctl/commands/dev.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 9d59411c046a..6a5d7968ec97 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -125,9 +125,7 @@ def interactive(app, graphscope_repo): """Build Interactive for high throughput scenarios""" if graphscope_repo is None: graphscope_repo = default_graphscope_repo_path - interactive_build_dir = os.path.join( - graphscope_repo, "k8s" - ) + interactive_build_dir = os.path.join(graphscope_repo, "k8s") if not os.path.exists(interactive_build_dir) or not os.path.isdir( interactive_build_dir ): From 4fcd3d63359ae77bc134e74c7858f47978fcddc3 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 6 Aug 2024 20:48:22 +0800 Subject: [PATCH 05/10] fixing errors Committed-by: xiaolei.zl from Dev container --- .github/workflows/nightly.yml | 2 +- k8s/dockerfiles/graphscope-dev.Dockerfile | 2 +- k8s/dockerfiles/manylinux2014-ext.Dockerfile | 11 ++++++++++- k8s/dockerfiles/manylinux2014.Dockerfile | 2 +- k8s/dockerfiles/vineyard-dev.Dockerfile | 2 +- k8s/internal/dataset.Dockerfile | 3 +++ 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1d2fa34f47c6..d02ff6ee02ca 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: fi sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - python3 -m pip install click + python3 -m pip install click packaging python3 gsctl.py install-deps dev --v6d-version v0.22.0 - name: Setup tmate session diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index b3df8bef7c6d..0cddb110991e 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -30,7 +30,7 @@ WORKDIR /home/graphscope COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl ARG VINEYARD_VERSION=main RUN cd /home/graphscope/gsctl && \ - python3 -m pip install click && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ cd /home/graphscope && \ rm -fr gsctl diff --git a/k8s/dockerfiles/manylinux2014-ext.Dockerfile b/k8s/dockerfiles/manylinux2014-ext.Dockerfile index 80fbbeeed3c6..5475f77affeb 100644 --- a/k8s/dockerfiles/manylinux2014-ext.Dockerfile +++ b/k8s/dockerfiles/manylinux2014-ext.Dockerfile @@ -6,6 +6,11 @@ FROM centos:7 AS builder # shanghai zoneinfo ENV TZ=Asia/Shanghai + +RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; + RUN yum install sudo -y && \ yum update glibc-common -y && \ sudo localedef -i en_US -f UTF-8 en_US.UTF-8 && \ @@ -18,7 +23,7 @@ ENV LANG=en_US.utf-8 COPY gsctl ./gsctl RUN cd ./gsctl && \ - python3 -m pip install click && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ rm -fr /root/gsctl @@ -36,6 +41,10 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then \ FROM centos:7 +RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; + COPY --from=builder /opt/graphscope /opt/graphscope COPY --from=builder /opt/openmpi /opt/openmpi COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index c8ba01c6930d..a5c119970ede 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -57,7 +57,7 @@ WORKDIR /home/graphscope COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl ARG VINEYARD_VERSION=main RUN cd /home/graphscope/gsctl && \ - sudo python3 -m pip install click && \ + sudo python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl && \ sudo yum clean all -y && \ diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index 7ef99f9a1a53..76bac2388dc8 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -42,7 +42,7 @@ COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ cd /home/graphscope/gsctl && \ - python3 -m pip install click && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl diff --git a/k8s/internal/dataset.Dockerfile b/k8s/internal/dataset.Dockerfile index e2131110e151..a4946f86e4cb 100644 --- a/k8s/internal/dataset.Dockerfile +++ b/k8s/internal/dataset.Dockerfile @@ -1,5 +1,8 @@ FROM centos:7 +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* +RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* + RUN yum localinstall -y https://github.com/aliyun/ossfs/releases/download/v1.80.5/ossfs_1.80.5_centos7.0_x86_64.rpm && \ yum clean all && \ rm -fr /var/cache/yum From 75d727982ca7a6997a6caf8cf3c439604886a67d Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 6 Aug 2024 21:10:21 +0800 Subject: [PATCH 06/10] fixing docker building Committed-by: xiaolei.zl from Dev container --- k8s/Makefile | 7 ++----- k8s/dockerfiles/graphscope-dev.Dockerfile | 6 +++--- k8s/dockerfiles/manylinux2014-ext.Dockerfile | 6 +++--- k8s/dockerfiles/manylinux2014.Dockerfile | 6 +++--- k8s/dockerfiles/vineyard-dev.Dockerfile | 6 +++--- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/k8s/Makefile b/k8s/Makefile index 14b091feba07..0f713c327f36 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -42,8 +42,7 @@ all: coordinator analytical interactive learning graphscope: all dev-wheel: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)../ && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ @@ -52,15 +51,13 @@ dev-wheel: manylinux2014-ext: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/.. && \ docker build \ -t graphscope/manylinux2014:ext-${ARCH} \ -f $(DOCKERFILES_DIR)/manylinux2014-ext.Dockerfile . graphscope-dev: cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index 0cddb110991e..2c25c59ebb0a 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -27,13 +27,13 @@ RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /op USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/gsctl && \ +RUN cd /home/graphscope/GraphScope/python/graphscope/gsctl && \ python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ cd /home/graphscope && \ - rm -fr gsctl + rm -fr GraphScope RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc RUN python3 -m pip --no-cache install pyyaml ipython --user diff --git a/k8s/dockerfiles/manylinux2014-ext.Dockerfile b/k8s/dockerfiles/manylinux2014-ext.Dockerfile index 5475f77affeb..965adb07f47b 100644 --- a/k8s/dockerfiles/manylinux2014-ext.Dockerfile +++ b/k8s/dockerfiles/manylinux2014-ext.Dockerfile @@ -21,11 +21,11 @@ RUN yum install sudo -y && \ ENV LC_ALL=en_US.utf-8 ENV LANG=en_US.utf-8 -COPY gsctl ./gsctl -RUN cd ./gsctl && \ +COPY . ./GraphScope +RUN cd ./GraphScope/python/graphscope/gsctl && \ python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ - rm -fr /root/gsctl + rm -fr /root/GraphScope # install hadoop for processing hadoop data source RUN if [ "$(uname -m)" = "aarch64" ]; then \ diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index a5c119970ede..b0854aada641 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -54,12 +54,12 @@ RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /op USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/gsctl && \ +RUN cd /home/graphscope/GraphScope/python/graphscope/gsctl && \ sudo python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl && \ + cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope && \ sudo yum clean all -y && \ sudo rm -fr /var/cache/yum RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index 76bac2388dc8..b2e521cbf667 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -38,13 +38,13 @@ RUN mkdir -p /var/log/graphscope && chown -R graphscope:graphscope /var/log/grap USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ - cd /home/graphscope/gsctl && \ + cd /home/graphscope/GraphScope/python/graphscope/gsctl && \ python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl + cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope RUN python3 -m pip --no-cache install pyyaml --user From 6ea462ff2dc03b6b210d3b8a9bb2f873dc45f521 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 7 Aug 2024 10:21:15 +0800 Subject: [PATCH 07/10] fixing Committed-by: xiaolei.zl from Dev container --- k8s/dockerfiles/manylinux2014.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index b0854aada641..20749c03cddf 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -41,6 +41,10 @@ RUN chmod +x /opt/graphscope/bin/* /opt/openmpi/bin/* RUN useradd -m graphscope -u 1001 \ && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; + # Install jdk-11 RUN yum install -y sudo vim && \ yum install python3-pip -y && \ From 6ebeb3e2c1a85889b9778a7044537d1b539bdfe8 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 7 Aug 2024 10:38:08 +0800 Subject: [PATCH 08/10] fixing Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container --- k8s/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/k8s/Makefile b/k8s/Makefile index 0f713c327f36..c2902803664b 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -57,7 +57,7 @@ manylinux2014-ext: -f $(DOCKERFILES_DIR)/manylinux2014-ext.Dockerfile . graphscope-dev: - cd $(WORKING_DIR) && \ + cd $(WORKING_DIR)/.. && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ @@ -65,8 +65,7 @@ graphscope-dev: -f $(DOCKERFILES_DIR)/graphscope-dev.Dockerfile . vineyard-dev: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/.. && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ From 0956669b2347665a07279483d3915c3cc72f5c3b Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 7 Aug 2024 16:26:34 +0800 Subject: [PATCH 09/10] revert changes Committed-by: xiaolei.zl from Dev container --- .github/workflows/nightly.yml | 2 +- k8s/Makefile | 13 ++++++++----- k8s/dockerfiles/graphscope-dev.Dockerfile | 8 ++++---- k8s/dockerfiles/manylinux2014-ext.Dockerfile | 17 ++++------------- k8s/dockerfiles/manylinux2014.Dockerfile | 12 ++++-------- k8s/dockerfiles/vineyard-dev.Dockerfile | 8 ++++---- k8s/internal/dataset.Dockerfile | 3 --- .../gsctl/scripts/make_image_command.sh | 6 ------ 8 files changed, 25 insertions(+), 44 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d02ff6ee02ca..b6d18058fa74 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: fi sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - python3 -m pip install click packaging + python3 -m pip install click python3 gsctl.py install-deps dev --v6d-version v0.22.0 - name: Setup tmate session diff --git a/k8s/Makefile b/k8s/Makefile index c2902803664b..84c829324f30 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -42,7 +42,8 @@ all: coordinator analytical interactive learning graphscope: all dev-wheel: - cd $(WORKING_DIR)../ && \ + cd $(WORKING_DIR) && \ + cd ../python/graphscope && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ @@ -51,13 +52,15 @@ dev-wheel: manylinux2014-ext: - cd $(WORKING_DIR)/.. && \ + cd $(WORKING_DIR) && \ + cd ../python/graphscope && \ docker build \ -t graphscope/manylinux2014:ext-${ARCH} \ -f $(DOCKERFILES_DIR)/manylinux2014-ext.Dockerfile . graphscope-dev: - cd $(WORKING_DIR)/.. && \ + cd $(WORKING_DIR) && \ + cd ../python/graphscope && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ @@ -65,7 +68,8 @@ graphscope-dev: -f $(DOCKERFILES_DIR)/graphscope-dev.Dockerfile . vineyard-dev: - cd $(WORKING_DIR)/.. && \ + cd $(WORKING_DIR) && \ + cd ../python/graphscope && \ docker build \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ @@ -146,7 +150,6 @@ interactive-experimental: -t graphscope/interactive-experimental:${VERSION} \ -f $(DOCKERFILES_DIR)/interactive-experimental.Dockerfile . - # flex interactive flex-interactive: cd $(WORKING_DIR)/.. && \ diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index 2c25c59ebb0a..b3df8bef7c6d 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -27,13 +27,13 @@ RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /op USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope +COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/GraphScope/python/graphscope/gsctl && \ - python3 -m pip install click packaging && \ +RUN cd /home/graphscope/gsctl && \ + python3 -m pip install click && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ cd /home/graphscope && \ - rm -fr GraphScope + rm -fr gsctl RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc RUN python3 -m pip --no-cache install pyyaml ipython --user diff --git a/k8s/dockerfiles/manylinux2014-ext.Dockerfile b/k8s/dockerfiles/manylinux2014-ext.Dockerfile index 965adb07f47b..80fbbeeed3c6 100644 --- a/k8s/dockerfiles/manylinux2014-ext.Dockerfile +++ b/k8s/dockerfiles/manylinux2014-ext.Dockerfile @@ -6,11 +6,6 @@ FROM centos:7 AS builder # shanghai zoneinfo ENV TZ=Asia/Shanghai - -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; - RUN yum install sudo -y && \ yum update glibc-common -y && \ sudo localedef -i en_US -f UTF-8 en_US.UTF-8 && \ @@ -21,11 +16,11 @@ RUN yum install sudo -y && \ ENV LC_ALL=en_US.utf-8 ENV LANG=en_US.utf-8 -COPY . ./GraphScope -RUN cd ./GraphScope/python/graphscope/gsctl && \ - python3 -m pip install click packaging && \ +COPY gsctl ./gsctl +RUN cd ./gsctl && \ + python3 -m pip install click && \ python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ - rm -fr /root/GraphScope + rm -fr /root/gsctl # install hadoop for processing hadoop data source RUN if [ "$(uname -m)" = "aarch64" ]; then \ @@ -41,10 +36,6 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then \ FROM centos:7 -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; - COPY --from=builder /opt/graphscope /opt/graphscope COPY --from=builder /opt/openmpi /opt/openmpi COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index 20749c03cddf..c8ba01c6930d 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -41,10 +41,6 @@ RUN chmod +x /opt/graphscope/bin/* /opt/openmpi/bin/* RUN useradd -m graphscope -u 1001 \ && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; - # Install jdk-11 RUN yum install -y sudo vim && \ yum install python3-pip -y && \ @@ -58,12 +54,12 @@ RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /op USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope +COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/GraphScope/python/graphscope/gsctl && \ - sudo python3 -m pip install click packaging && \ +RUN cd /home/graphscope/gsctl && \ + sudo python3 -m pip install click && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope && \ + cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl && \ sudo yum clean all -y && \ sudo rm -fr /var/cache/yum RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index b2e521cbf667..7ef99f9a1a53 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -38,13 +38,13 @@ RUN mkdir -p /var/log/graphscope && chown -R graphscope:graphscope /var/log/grap USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope +COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ - cd /home/graphscope/GraphScope/python/graphscope/gsctl && \ - python3 -m pip install click packaging && \ + cd /home/graphscope/gsctl && \ + python3 -m pip install click && \ python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope + cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl RUN python3 -m pip --no-cache install pyyaml --user diff --git a/k8s/internal/dataset.Dockerfile b/k8s/internal/dataset.Dockerfile index a4946f86e4cb..e2131110e151 100644 --- a/k8s/internal/dataset.Dockerfile +++ b/k8s/internal/dataset.Dockerfile @@ -1,8 +1,5 @@ FROM centos:7 -RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* -RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* - RUN yum localinstall -y https://github.com/aliyun/ossfs/releases/download/v1.80.5/ossfs_1.80.5_centos7.0_x86_64.rpm && \ yum clean all && \ rm -fr /var/cache/yum diff --git a/python/graphscope/gsctl/scripts/make_image_command.sh b/python/graphscope/gsctl/scripts/make_image_command.sh index fe546bc5f8ba..d83ca19b649f 100644 --- a/python/graphscope/gsctl/scripts/make_image_command.sh +++ b/python/graphscope/gsctl/scripts/make_image_command.sh @@ -71,12 +71,6 @@ make_interactive-executor() { make interactive-executor REGISTRY=${registry} VERSION=${tag} } -# make the new flex interactive -make_flex-interactive() { - cd k8s - make flex-interactive REGISTRY=${registry} VERSION=${tag} -} - make_learning() { cd k8s make learning REGISTRY=${registry} VERSION=${tag} From 5782312e19cbf23827f617c892254fba3d350a5e Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 7 Aug 2024 16:27:13 +0800 Subject: [PATCH 10/10] minor Committed-by: xiaolei.zl from Dev container --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b6d18058fa74..1d2fa34f47c6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: fi sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - python3 -m pip install click + python3 -m pip install click python3 gsctl.py install-deps dev --v6d-version v0.22.0 - name: Setup tmate session