From 14ead9116cb48976db8013169be7965b95eaa4ac Mon Sep 17 00:00:00 2001 From: catpineapple Date: Mon, 9 Dec 2024 18:52:37 +0800 Subject: [PATCH 1/5] add disable query and load for be restart --- docker/runtime/be/resource/be_entrypoint.sh | 12 +++ docker/runtime/be/resource/be_prestop.sh | 92 +++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/docker/runtime/be/resource/be_entrypoint.sh b/docker/runtime/be/resource/be_entrypoint.sh index 0d83ba780a49bd..3377ffd4c6b3eb 100755 --- a/docker/runtime/be/resource/be_entrypoint.sh +++ b/docker/runtime/be/resource/be_entrypoint.sh @@ -107,6 +107,17 @@ show_backends(){ echo "$backends" } +enable_query_and_load(){ + local svc=$1 + set_enable_query_and_load=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\",\"disable_load\"=\"false\");" 2>&1` + log_stderr "[info] use root no password set enable query and load result $set_enable_query_and_load ." + if echo $set_enable_query_and_load | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "[info] use username and password that configured to enable query and load." + set_enable_query_and_load=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\",\"disable_load\"=\"false\");"` + fi + log_stderr "[info] set enable query and load return $set_enable_query_and_load" +} + # get all registered fe in cluster, for check the fe have `MASTER`. function show_frontends() { @@ -157,6 +168,7 @@ add_self() memlist=`show_backends $svc` if echo "$memlist" | grep -q -w "$MY_SELF" &>/dev/null ; then log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) exist in FE, start be directly ..." + enable_query_and_load $svc break; fi diff --git a/docker/runtime/be/resource/be_prestop.sh b/docker/runtime/be/resource/be_prestop.sh index 798cf28944fb37..31a8d7d66ffb79 100755 --- a/docker/runtime/be/resource/be_prestop.sh +++ b/docker/runtime/be/resource/be_prestop.sh @@ -18,4 +18,96 @@ DORIS_ROOT=${DORIS_ROOT:-"/opt/apache-doris"} DORIS_HOME=${DORIS_ROOT}/be + +BE_CONFIG=$DORIS_HOME/conf/be.conf + +AUTH_PATH="/etc/basic_auth" + +DB_ADMIN_USER=${USER:-"root"} + +DB_ADMIN_PASSWD=$PASSWD + +ENV_FE_ADDR=$ENV_FE_ADDR +FE_QUERY_PORT=${FE_QUERY_PORT:-9030} + + +HEARTBEAT_PORT=9050 +MY_SELF= +MY_IP=`hostname -i` +MY_HOSTNAME=`hostname -f` + +log_stderr() +{ + echo "[`date`] $@" >&2 +} + +resolve_password_from_secret() +{ + if [[ -f "$AUTH_PATH/password" ]]; then + DB_ADMIN_PASSWD=`cat $AUTH_PATH/password` + fi + if [[ -f "$AUTH_PATH/username" ]]; then + DB_ADMIN_USER=`cat $AUTH_PATH/username` + fi +} + +parse_confval_from_conf() +{ + # a naive script to grep given confkey from fe conf file + # assume conf format: ^\s*\s*=\s*\s*$ + local confkey=$1 + local confvalue=`grep "\<$confkey\>" $BE_CONFIG | grep -v '^\s*#' | sed 's|^\s*'$confkey'\s*=\s*\(.*\)\s*$|\1|g'` + echo "$confvalue" +} + + +collect_env_info() +{ + # heartbeat_port from conf file + local heartbeat_port=`parse_confval_from_conf "heartbeat_service_port"` + if [[ "x$heartbeat_port" != "x" ]] ; then + HEARTBEAT_PORT=$heartbeat_port + fi + + if [[ "x$HOST_TYPE" == "xIP" ]] ; then + MY_SELF=$MY_IP + else + MY_SELF=$MY_HOSTNAME + fi +} + +function show_frontends() +{ + local addr=$1 + frontends=`timeout 15 mysql --connect-timeout 2 -h $addr -P $FE_QUERY_PORT -uroot --batch -e 'show frontends;' 2>&1` + log_stderr "[info] use root no password show frontends result $frontends ." + if echo $frontends | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "[info] use username and passwore that configured to show frontends." + frontends=`timeout 15 mysql --connect-timeout 2 -h $addr -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --batch -e 'show frontends;'` + fi + + echo "$frontends" +} + +disable_query_and_load(){ + local svc=$1 + disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\",\"disable_load\"=\"true\");" 2>&1` + if echo $disable_result | grep -w "1045" | grep -q -w "28000" &>/dev/null ; then + log_stderr "disable_query_and_load use pwd to run 'ALTER SYSTEM MODIFY BACKEND' sql ." + disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\",\"disable_load\"=\"true\");" 2>&1` + fi + if [[ "x$disable_result" == "x" ]] ; then + log_stderr "disable_query_and_load success ." + else + log_stderr "disable_query_and_load failed: $disable_result ." + fi +} + +resolve_password_from_secret +collect_env_info +disable_query_and_load $ENV_FE_ADDR + + +echo "===================================end====================================" + $DORIS_HOME/bin/stop_be.sh --grace From 2950cdabf1a2883499a53d1fa8b3c35fa07cc351 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Mon, 9 Dec 2024 19:04:30 +0800 Subject: [PATCH 2/5] add disable query and load for be restart --- docker/runtime/be/resource/be_prestop.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/runtime/be/resource/be_prestop.sh b/docker/runtime/be/resource/be_prestop.sh index 31a8d7d66ffb79..0578e4d4eb0b41 100755 --- a/docker/runtime/be/resource/be_prestop.sh +++ b/docker/runtime/be/resource/be_prestop.sh @@ -107,7 +107,4 @@ resolve_password_from_secret collect_env_info disable_query_and_load $ENV_FE_ADDR - -echo "===================================end====================================" - $DORIS_HOME/bin/stop_be.sh --grace From d3119331c8ebfb5cec68397d45da5692725f6cbf Mon Sep 17 00:00:00 2001 From: catpineapple Date: Tue, 24 Dec 2024 15:25:32 +0800 Subject: [PATCH 3/5] disable query and load for be restart failed retry --- docker/runtime/be/resource/be_entrypoint.sh | 32 ++++++++++- docker/runtime/be/resource/be_prestop.sh | 62 +++++++++++++++++++-- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/docker/runtime/be/resource/be_entrypoint.sh b/docker/runtime/be/resource/be_entrypoint.sh index 3377ffd4c6b3eb..04f0fac26306d5 100755 --- a/docker/runtime/be/resource/be_entrypoint.sh +++ b/docker/runtime/be/resource/be_entrypoint.sh @@ -23,6 +23,8 @@ FE_QUERY_PORT=${FE_QUERY_PORT:-9030} PROBE_TIMEOUT=60 # interval time to probe fe. PROBE_INTERVAL=2 +NEED_PRE_STOP=${NEED_PRE_STOP:-"false"} + # rpc port for fe communicate with be. HEARTBEAT_PORT=9050 # fqdn or ip @@ -112,7 +114,7 @@ enable_query_and_load(){ set_enable_query_and_load=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\",\"disable_load\"=\"false\");" 2>&1` log_stderr "[info] use root no password set enable query and load result $set_enable_query_and_load ." if echo $set_enable_query_and_load | grep -w "1045" | grep -q -w "28000" &>/dev/null; then - log_stderr "[info] use username and password that configured to enable query and load." + log_stderr "[info] use username($DB_ADMIN_USER) and password that configured to enable query and load." set_enable_query_and_load=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\",\"disable_load\"=\"false\");"` fi log_stderr "[info] set enable query and load return $set_enable_query_and_load" @@ -157,6 +159,17 @@ collect_env_info() fi } +check_enable(){ + local memlist=$1 + if echo "$memlist" | grep "$MY_SELF" | grep "isQueryDisabled\":false" | grep -q -w "isLoadDisabled\":false" &>/dev/null ; then + log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) enable_query_and_load success " + echo "true" + else + log_stderr "[error] Check myself ($MY_SELF:$HEARTBEAT_PORT) enble_query_and_load failed " + echo "false" + fi +} + add_self() { local svc=$1 @@ -168,8 +181,23 @@ add_self() memlist=`show_backends $svc` if echo "$memlist" | grep -q -w "$MY_SELF" &>/dev/null ; then log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) exist in FE, start be directly ..." + if [[ "x$NEED_PRE_STOP" == "xfalse" ]] ; then + break; + fi enable_query_and_load $svc - break; + res=`check_enable $memlist` + if [[ "x$res" == "xtrue" ]] ; then + break; + else + sleep $PROBE_INTERVAL + let "expire=start+timeout" + now=`date +%s` + if [[ $expire -le $now ]] ; then + log_stderr "[error] exit probe master for probing timeout." + return 0 + fi + continue; + fi fi # check fe cluster have master, if fe have not master wait. diff --git a/docker/runtime/be/resource/be_prestop.sh b/docker/runtime/be/resource/be_prestop.sh index 0578e4d4eb0b41..0cb9bb0bf3ab86 100755 --- a/docker/runtime/be/resource/be_prestop.sh +++ b/docker/runtime/be/resource/be_prestop.sh @@ -19,6 +19,11 @@ DORIS_ROOT=${DORIS_ROOT:-"/opt/apache-doris"} DORIS_HOME=${DORIS_ROOT}/be +# interval time. +PROBE_INTERVAL=5 +RETRY_TIMES=10 +NEED_PRE_STOP=${NEED_PRE_STOP:-"false"} + BE_CONFIG=$DORIS_HOME/conf/be.conf AUTH_PATH="/etc/basic_auth" @@ -89,22 +94,67 @@ function show_frontends() echo "$frontends" } +show_backends(){ + local svc=$1 + backends=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e 'SHOW BACKENDS;' 2>&1` + log_stderr "[info] use root no password show backends result $backends ." + if echo $backends | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "[info] use username and password that configured to show backends." + backends=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e 'SHOW BACKENDS;'` + fi + + echo "$backends" +} + + disable_query_and_load(){ local svc=$1 disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\",\"disable_load\"=\"true\");" 2>&1` if echo $disable_result | grep -w "1045" | grep -q -w "28000" &>/dev/null ; then - log_stderr "disable_query_and_load use pwd to run 'ALTER SYSTEM MODIFY BACKEND' sql ." + log_stderr "[info] disable_query_and_load use pwd to run 'ALTER SYSTEM MODIFY BACKEND' sql ." disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\",\"disable_load\"=\"true\");" 2>&1` fi if [[ "x$disable_result" == "x" ]] ; then - log_stderr "disable_query_and_load success ." + log_stderr "[info] disable_query_and_load success ." else - log_stderr "disable_query_and_load failed: $disable_result ." + log_stderr "[error] disable_query_and_load failed: $disable_result ." fi } -resolve_password_from_secret -collect_env_info -disable_query_and_load $ENV_FE_ADDR +check_disable(){ + local svc=$1 + memlist=`show_backends $svc` + if echo "$memlist" | grep "$MY_SELF" | grep "isQueryDisabled\":true" | grep -q -w "isLoadDisabled\":true" &>/dev/null ; then + log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) disable_query_and_load success " + echo "true" + else + log_stderr "[error] Check myself ($MY_SELF:$HEARTBEAT_PORT) disable_query_and_load failed " + echo "false" + fi +} + +# disable query/load and check for stop +prepare_stop(){ + local svc=$1 + for ((i=1; i<=RETRY_TIMES; i++)) + do + disable_query_and_load $ENV_FE_ADDR + disable_res=`check_disable $ENV_FE_ADDR` + if [[ "x$disable_res" == "xtrue" ]] ; then + break + else + log_stderr "[error] will retry to set be disable query and load " + sleep $PROBE_INTERVAL + fi + done +} + + +if [[ "x$NEED_PRE_STOP" == "xtrue" ]] ; then + resolve_password_from_secret + collect_env_info + prepare_stop $ENV_FE_ADDR +fi + $DORIS_HOME/bin/stop_be.sh --grace From 9a62d6823c19d36b334414fb0e0c3c005577c95e Mon Sep 17 00:00:00 2001 From: catpineapple Date: Tue, 24 Dec 2024 15:29:17 +0800 Subject: [PATCH 4/5] disable query and load for be restart failed retry --- docker/runtime/be/resource/be_entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/runtime/be/resource/be_entrypoint.sh b/docker/runtime/be/resource/be_entrypoint.sh index 04f0fac26306d5..91181e767913e5 100755 --- a/docker/runtime/be/resource/be_entrypoint.sh +++ b/docker/runtime/be/resource/be_entrypoint.sh @@ -23,7 +23,7 @@ FE_QUERY_PORT=${FE_QUERY_PORT:-9030} PROBE_TIMEOUT=60 # interval time to probe fe. PROBE_INTERVAL=2 -NEED_PRE_STOP=${NEED_PRE_STOP:-"false"} +NEED_PRE_START=${NEED_PRE_STOP:-"false"} # rpc port for fe communicate with be. HEARTBEAT_PORT=9050 @@ -181,7 +181,7 @@ add_self() memlist=`show_backends $svc` if echo "$memlist" | grep -q -w "$MY_SELF" &>/dev/null ; then log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) exist in FE, start be directly ..." - if [[ "x$NEED_PRE_STOP" == "xfalse" ]] ; then + if [[ "x$NEED_PRE_START" == "xfalse" ]] ; then break; fi enable_query_and_load $svc From 6038e6a73a2a994739faf9a8ef12978065e8b491 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Tue, 24 Dec 2024 16:02:26 +0800 Subject: [PATCH 5/5] disable query for be restart failed retry --- docker/runtime/be/resource/be_entrypoint.sh | 22 +++++++++---------- docker/runtime/be/resource/be_prestop.sh | 24 ++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docker/runtime/be/resource/be_entrypoint.sh b/docker/runtime/be/resource/be_entrypoint.sh index 91181e767913e5..d516f448338ea8 100755 --- a/docker/runtime/be/resource/be_entrypoint.sh +++ b/docker/runtime/be/resource/be_entrypoint.sh @@ -109,15 +109,15 @@ show_backends(){ echo "$backends" } -enable_query_and_load(){ +enable_query(){ local svc=$1 - set_enable_query_and_load=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\",\"disable_load\"=\"false\");" 2>&1` - log_stderr "[info] use root no password set enable query and load result $set_enable_query_and_load ." - if echo $set_enable_query_and_load | grep -w "1045" | grep -q -w "28000" &>/dev/null; then - log_stderr "[info] use username($DB_ADMIN_USER) and password that configured to enable query and load." - set_enable_query_and_load=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\",\"disable_load\"=\"false\");"` + set_enable_query=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\");" 2>&1` + log_stderr "[info] use root no password set enable query result $set_enable_query ." + if echo $set_enable_query | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "[info] use username($DB_ADMIN_USER) and password that configured to enable query." + set_enable_query=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"false\");"` fi - log_stderr "[info] set enable query and load return $set_enable_query_and_load" + log_stderr "[info] set enable query return $set_enable_query" } # get all registered fe in cluster, for check the fe have `MASTER`. @@ -161,11 +161,11 @@ collect_env_info() check_enable(){ local memlist=$1 - if echo "$memlist" | grep "$MY_SELF" | grep "isQueryDisabled\":false" | grep -q -w "isLoadDisabled\":false" &>/dev/null ; then - log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) enable_query_and_load success " + if echo "$memlist" | grep "$MY_SELF" | grep -q -w "isQueryDisabled\":false" &>/dev/null ; then + log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) enable_query success " echo "true" else - log_stderr "[error] Check myself ($MY_SELF:$HEARTBEAT_PORT) enble_query_and_load failed " + log_stderr "[error] Check myself ($MY_SELF:$HEARTBEAT_PORT) enble_query failed " echo "false" fi } @@ -184,7 +184,7 @@ add_self() if [[ "x$NEED_PRE_START" == "xfalse" ]] ; then break; fi - enable_query_and_load $svc + enable_quer $svc res=`check_enable $memlist` if [[ "x$res" == "xtrue" ]] ; then break; diff --git a/docker/runtime/be/resource/be_prestop.sh b/docker/runtime/be/resource/be_prestop.sh index 0cb9bb0bf3ab86..0a75988ac84bea 100755 --- a/docker/runtime/be/resource/be_prestop.sh +++ b/docker/runtime/be/resource/be_prestop.sh @@ -107,43 +107,43 @@ show_backends(){ } -disable_query_and_load(){ +disable_query(){ local svc=$1 - disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\",\"disable_load\"=\"true\");" 2>&1` + disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\");" 2>&1` if echo $disable_result | grep -w "1045" | grep -q -w "28000" &>/dev/null ; then - log_stderr "[info] disable_query_and_load use pwd to run 'ALTER SYSTEM MODIFY BACKEND' sql ." - disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\",\"disable_load\"=\"true\");" 2>&1` + log_stderr "[info] disable_query use pwd to run 'ALTER SYSTEM MODIFY BACKEND' sql ." + disable_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM MODIFY BACKEND \"$MY_SELF:$HEARTBEAT_PORT\" SET (\"disable_query\" = \"true\");" 2>&1` fi if [[ "x$disable_result" == "x" ]] ; then - log_stderr "[info] disable_query_and_load success ." + log_stderr "[info] disable_query success ." else - log_stderr "[error] disable_query_and_load failed: $disable_result ." + log_stderr "[error] disable_query failed: $disable_result ." fi } check_disable(){ local svc=$1 memlist=`show_backends $svc` - if echo "$memlist" | grep "$MY_SELF" | grep "isQueryDisabled\":true" | grep -q -w "isLoadDisabled\":true" &>/dev/null ; then - log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) disable_query_and_load success " + if echo "$memlist" | grep "$MY_SELF" | grep -q -w "isQueryDisabled\":true" &>/dev/null ; then + log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) disable_query success " echo "true" else - log_stderr "[error] Check myself ($MY_SELF:$HEARTBEAT_PORT) disable_query_and_load failed " + log_stderr "[error] Check myself ($MY_SELF:$HEARTBEAT_PORT) disable_query failed " echo "false" fi } -# disable query/load and check for stop +# disable query and check for stop prepare_stop(){ local svc=$1 for ((i=1; i<=RETRY_TIMES; i++)) do - disable_query_and_load $ENV_FE_ADDR + disable_query $ENV_FE_ADDR disable_res=`check_disable $ENV_FE_ADDR` if [[ "x$disable_res" == "xtrue" ]] ; then break else - log_stderr "[error] will retry to set be disable query and load " + log_stderr "[error] will retry to set be disable query " sleep $PROBE_INTERVAL fi done