From 76d8eacfc6d910047c70dbe6aa5c2c30ff604020 Mon Sep 17 00:00:00 2001 From: Kent Dong Date: Tue, 20 Feb 2024 09:57:37 +0800 Subject: [PATCH] feat: Initialize base configs via API server (#58) --- compose/docker-compose.yml | 39 +++-- compose/scripts/init.sh | 202 +---------------------- compose/scripts/precheck.sh | 269 ------------------------------ compose/scripts/prepare.sh | 316 ++++++++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+), 482 deletions(-) mode change 100755 => 100644 compose/scripts/precheck.sh create mode 100644 compose/scripts/prepare.sh diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 25a2b42..57b710c 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -28,14 +28,9 @@ services: image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/runner:${HIGRESS_RUNNER_TAG} command: - ./init.sh + env_file: + - ./.env profiles: [ "init" ] - environment: - - CONFIG_STORAGE - - NACOS_SERVER_URL - - NACOS_NS - - NACOS_USERNAME - - NACOS_PASSWORD - - NACOS_DATA_ENC_KEY networks: - higress-net volumes: @@ -47,14 +42,11 @@ services: image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/runner:${HIGRESS_RUNNER_TAG} command: - ./precheck.sh - environment: - - CONFIG_STORAGE - - NACOS_SERVER_URL - - NACOS_NS - - NACOS_USERNAME - - NACOS_PASSWORD + env_file: + - ./.env networks: - higress-net + restart: on-failure volumes: - ./volumes:/mnt/volumes:rw - ./scripts:/workspace:ro @@ -100,6 +92,23 @@ services: - ./volumes/api:/etc/api:ro - ${FILE_ROOT_DIR:-./volumes/dummy}:/opt/data/:rw + prepare: + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/runner:${HIGRESS_RUNNER_TAG} + command: + - ./prepare.sh + env_file: + - ./.env + depends_on: + apiserver: + condition: service_healthy + networks: + - higress-net + restart: on-failure + volumes: + - ./volumes:/mnt/volumes:rw + - ./scripts:/workspace:ro + - ${FILE_ROOT_DIR:-./volumes/dummy}:/opt/data/:ro + controller: image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress:${HIGRESS_CONTROLLER_TAG} command: @@ -111,8 +120,8 @@ services: env_file: - ./env/controller.env depends_on: - apiserver: - condition: service_healthy + prepare: + condition: service_completed_successfully networks: - higress-net restart: always diff --git a/compose/scripts/init.sh b/compose/scripts/init.sh index 0291ce4..9bc74cc 100755 --- a/compose/scripts/init.sh +++ b/compose/scripts/init.sh @@ -26,78 +26,6 @@ checkExitCode() { fi } -publishConfig() { - # $1 namespace - # $2 configType: plural - # $3 configName - # $4 content - # $5 skipWhenExisted - case $CONFIG_STORAGE in - nacos) - publishNacosConfig "$@" - ;; - file) - publishFileConfig "$@" - ;; - *) - printf " Unknown storage type: %s\n" "$CONFIG_STORAGE" - exit -1 - ;; - esac -} - -publishNacosConfig() { - # $1 namespace - # $2 configType: plural - # $3 configName - # $4 content - # $5 skipWhenExisted - local group="$1" - local dataId="$2.$3" - local content="$4" - local skipWhenExisted=$5 - - if [ "$skipWhenExisted" == true ]; then - statusCode=$(curl -s -o /dev/null -w "%{http_code}" "${NACOS_SERVER_URL}/v1/cs/configs?accessToken=${NACOS_ACCESS_TOKEN}&tenant=${NACOS_NS}&dataId=${dataId}&group=${group}") - if [ $statusCode -eq 200 ]; then - echo " Config $group/$dataId already exists in namespace ${NACOS_NS}" - return 0 - elif [ $statusCode -ne 404 ]; then - echo " Checking config $group/$dataId in tenant ${NACOS_NS} failed with $statusCode" - exit -1 - fi - fi - - statusCode="$(curl -s -o /dev/null -w "%{http_code}" "${NACOS_SERVER_URL}/v1/cs/configs?accessToken=${NACOS_ACCESS_TOKEN}" --data-urlencode "tenant=${NACOS_NS}" --data-urlencode "dataId=${dataId}" --data-urlencode "group=${group}" --data-urlencode "content=${content}")" - if [ $statusCode -ne 200 ]; then - echo " Publishing config ${group}/${dataId} in tenant ${NACOS_NS} failed with $statusCode" - exit -1 - fi - return 0 -} - -publishFileConfig() { - # $1 namespace: ignored. only for alignment - # $2 configType: plural - # $3 configName - # $4 content - # $5 skipWhenExisted - local configDir="${FILE_ROOT_DIR}/$2" - local configFile="${configDir}/$3.yaml" - local content="$4" - local skipWhenExisted=$5 - - if [ "$skipWhenExisted" == true ] && [ -f "$configFile" ]; then - echo " Config file [$configFile] already exists" - return 0 - fi - - mkdir -p "$configDir" - checkExitCode " Creating config directory [$configDir] fails with $?" - echo "$content" > "$configFile" - return 0 -} - initializeConfigStorage() { CONFIG_STORAGE=${CONFIG_STORAGE:-nacos} @@ -249,8 +177,10 @@ initializeController() { mkdir -p $VOLUMES_ROOT/controller && cd "$_" - mkdir -p ./log/nacos - chmod a+w ./log/nacos + if [ "$CONFIG_STORAGE" == "nacos" ]; then + mkdir -p ./log/nacos + chmod a+w ./log/nacos + fi } initializePilot() { @@ -328,40 +258,6 @@ EOF checkExitCode "Generating certificate for gateway fails with $?" chmod a+r gateway-key.pem fi - - read -r -d '' content << EOF -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app: higress-gateway - higress: higress-system-higress-gateway - name: higress-config - namespace: higress-system -data: - mesh: |- - accessLogEncoding: TEXT - accessLogFile: /dev/stdout - accessLogFormat: | - {"authority":"%REQ(:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"} - configSources: - - address: xds://controller:15051 - defaultConfig: - disableAlpnH2: true - discoveryAddress: pilot:15012 - proxyStatsMatcher: - inclusionRegexps: - - .* - dnsRefreshRate: 200s - enableAutoMtls: false - enablePrometheusMerge: true - ingressControllerMode: "OFF" - protocolDetectionTimeout: 100ms - rootNamespace: higress-system - trustDomain: cluster.local - meshNetworks: 'networks: {}' -EOF - publishConfig "higress-system" "configmaps" "higress-config" "$content" true } initializeGateway() { @@ -379,98 +275,8 @@ initializeGateway() { mkdir -p $VOLUMES_ROOT/gateway/istio/data } -initializeMcpBridge() { - echo "Initializing McpBridge resource..." - - read -r -d '' mcpbridgeContent << EOF -apiVersion: networking.higress.io/v1 -kind: McpBridge -metadata: - creationTimestamp: "$(now)" - name: default - namespace: higress-system -spec: - registries: -EOF - - if [ "$CONFIG_STORAGE" == "nacos" ]; then - if [[ "$NACOS_SERVER_URL" =~ ^http://([a-zA-Z0-9.]+?)(:([0-9]+))/nacos$ ]]; then - NACOS_SERVER_DOMAIN="${BASH_REMATCH[1]}" - NACOS_SERVER_PORT="${BASH_REMATCH[3]}" - else - echo " Unable to parse Nacos server URL. Skip creating the McpBridge resource" - return - fi - - nacosAuthSecretName="" - - if [ -n "$NACOS_USERNAME" ] && [ -n "$NACOS_PASSWORD" ]; then - nacosAuthSecretName="nacos-auth-default" - read -r -d '' nacosAuthSecretContent << EOF -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: "$(now)" - name: ${nacosAuthSecretName} - namespace: higress-system -data: - nacosUsername: $(echo -n "${NACOS_USERNAME}" | base64 -w 0) - nacosPassword: $(echo -n "${NACOS_PASSWORD}" | base64 -w 0) -type: Opaque -EOF - publishConfig "higress-system" "secrets" "${nacosAuthSecretName}" "$nacosAuthSecretContent" true - fi - - read -r -d '' mcpbridgeContent << EOF -${mcpbridgeContent} - - domain: ${NACOS_SERVER_DOMAIN} - nacosGroups: - - DEFAULT_GROUP - nacosNamespaceId: "" - name: nacos - port: ${NACOS_SERVER_PORT:-80} - type: nacos2 - authSecretName: "${nacosAuthSecretName}" -EOF - fi - - publishConfig "higress-system" "mcpbridges" "default" "$mcpbridgeContent" true -} - -initializeConsole() { - echo "Initializing console configurations..." - - read -r -d '' content << EOF -apiVersion: v1 -kind: ConfigMap -metadata: - creationTimestamp: "$(now)" - name: higress-console - namespace: higress-system -data: - mode: standalone -EOF - publishConfig "higress-system" "configmaps" "higress-console" "$content" true - - read -r -d '' content << EOF -apiVersion: v1 -data: - iv: $(cat /dev/urandom | tr -dc '[:graph:]' | fold -w 16 | head -n 1 | tr -d '\n' | base64 -w 0) - key: $(cat /dev/urandom | tr -dc '[:graph:]' | fold -w 32 | head -n 1 | tr -d '\n' | base64 -w 0) -kind: Secret -metadata: - creationTimestamp: "$(now)" - name: higress-console - namespace: higress-system -type: Opaque -EOF - publishConfig "higress-system" "secrets" "higress-console" "$content" true -} - initializeConfigStorage initializeApiServer initializeController initializePilot initializeGateway -initializeMcpBridge -initializeConsole diff --git a/compose/scripts/precheck.sh b/compose/scripts/precheck.sh old mode 100755 new mode 100644 index 0d80eca..09bcbf1 --- a/compose/scripts/precheck.sh +++ b/compose/scripts/precheck.sh @@ -3,185 +3,6 @@ BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" VOLUMES_ROOT="/mnt/volumes" -RSA_KEY_LENGTH=4096 - -NACOS_SERVER_URL=${NACOS_SERVER_URL%/} -NACOS_ACCESS_TOKEN="" -FILE_ROOT_DIR="/opt/data" - -now() { - echo "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" -} - -checkExitCode() { - # $1 message - retVal=$? - if [ $retVal -ne 0 ]; then - echo ${1:-" Command fails with $retVal"} - exit $retVal - fi -} - -checkConfigExists() { - # $1 namespace - # $2 configType: plural - # $3 configName - case $CONFIG_STORAGE in - nacos) - checkNacosConfigExists "$@" - return $? - ;; - file) - checkFileConfigExists "$@" - return $? - ;; - *) - printf " Unknown storage type: %s\n" "$CONFIG_STORAGE" - exit -1 - ;; - esac -} - -checkNacosConfigExists() { - # $1 namespace - # $2 configType: plural - # $3 configName - local group="$1" - local dataId="$2.$3" - statusCode=$(curl -s -o /dev/null -w "%{http_code}" "${NACOS_SERVER_URL}/v1/cs/configs?accessToken=${NACOS_ACCESS_TOKEN}&tenant=${NACOS_NS}&dataId=${dataId}&group=${group}") - if [ $statusCode -eq 200 ]; then - return 0 - elif [ $statusCode -eq 404 ]; then - return -1 - else - echo " Checking config ${group}/${dataId} in namespace ${NACOS_NS} failed with ${statusCode}" - exit -1 - fi -} - -checkFileConfigExists() { - # $1 namespace: ignored. only for alignment - # $2 configType: plural - # $3 configName - local configFile="${FILE_ROOT_DIR}/$2/$3.yaml" - if [ -f "$configFile" ]; then - return 0 - else - return 1 - fi -} - -getConfig() { - # $1 namespace - # $2 configType: plural - # $3 configName - case $CONFIG_STORAGE in - nacos) - getNacosConfig "$@" - ;; - file) - getFileConfig "$@" - ;; - *) - printf " Unknown storage type: %s\n" "$CONFIG_STORAGE" - exit -1 - ;; - esac -} - -getNacosConfig() { - # $1 namespace - # $2 configType: plural - # $3 configName - local group="$1" - local dataId="$2.$3" - - config="" - tmpFile=$(mktemp /tmp/higress-precheck-nacos.XXXXXXXXX.cfg) - statusCode=$(curl -s -o "$tmpFile" -w "%{http_code}" "${NACOS_SERVER_URL}/v1/cs/configs?accessToken=${NACOS_ACCESS_TOKEN}&tenant=${NACOS_NS}&dataId=${dataId}&group=${group}") - if [ $statusCode -eq 200 ]; then - config=$(cat "$tmpFile") - rm "$tmpFile" - return 0 - elif [ $statusCode -eq 404 ]; then - config = "" - return -1 - else - echo ${1:-" Getting config ${group}/${dataId} in namespace ${NACOS_NS} failed with $retVal"} - exit -1 - fi -} - -getFileConfig() { - # $1 namespace: ignored. only for alignment - # $2 configType: plural - # $3 configName - local configFile="${FILE_ROOT_DIR}/$2/$3.yaml" - if [ -f "$configFile" ]; then - config=$(cat "$configFile") - return 0 - else - config = "" - return -1 - fi -} - -checkStorage() { - CONFIG_STORAGE=${CONFIG_STORAGE:-nacos} - - case $CONFIG_STORAGE in - nacos) - checkNacos - ;; - file) - checkConfigDir - ;; - *) - printf "Unsupported storage type: %s\n" "$CONFIG_STORAGE" - ;; - esac -} - -checkNacos() { - echo "Checking Nacos server..." - - maxWaitTime=180 - for (( i = 0; i < $maxWaitTime; i++ )) - do - statusCode=$(curl -s -o /dev/null -w "%{http_code}" "${NACOS_SERVER_URL}/") - if [ "$statusCode" -eq "200" ]; then - nacosReady=true - echo "Nacos is ready." - break - fi - echo "Waiting for Nacos to get ready..." - sleep 1 - done - - if [ "${nacosReady}" != "true" ]; then - echo "Nacos server doesn't get ready within ${maxWaitTime} seconds. Initialization failed." - exit -1 - fi - - if [ -n "$NACOS_USERNAME" ] && [ -n "$NACOS_PASSWORD" ]; then - NACOS_ACCESS_TOKEN="$(curl -s "${NACOS_SERVER_URL}/v1/auth/login" -X POST --data-urlencode "username=${NACOS_USERNAME}" --data-urlencode "password=${NACOS_PASSWORD}" | jq -rM '.accessToken')"; - if [ -z "$NACOS_ACCESS_TOKEN" ]; then - echo "Unable to retrieve access token from Nacos. Possible causes are:" - echo " 1. Incorrect username or password." - echo " 2. The target Nacos service doesn't have authentication enabled." - fi - fi - - nacosNamespaces=$(curl -s "${NACOS_SERVER_URL}/v1/console/namespaces?accessToken=${NACOS_ACCESS_TOKEN}") - if [[ "$nacosNamespaces" != *"\"namespace\":\"${NACOS_NS}\""* ]]; then - echo " Unable to find namespace ${NACOS_NS} in Nacos." - exit -1 - fi -} - -checkConfigDir() { - echo "Initializing Config Directory..." -} checkApiServer() { echo "Checking API server configurations..." @@ -200,104 +21,14 @@ checkApiServer() { echo " Server certificate files of API server are missing." exit -1 fi - if [ ! -f nacos.key ]; then - echo " The data encryption key file is missing." - exit -1 - fi if [ ! -f client.key ] || [ ! -f client.crt ]; then echo " Client certificate files of API server are missing." exit -1 fi - if [ ! -f $VOLUMES_ROOT/kube/config ]; then echo " The kubeconfig file to access API server is missing." exit -1 fi } -checkPilot() { - echo "Checking pilot configurations..." - - if [ ! -d "$VOLUMES_ROOT/pilot/" ]; then - echo " The volume of pilot is missing." - exit -1 - fi - - if [ ! -d "$VOLUMES_ROOT/pilot/cacerts/" ]; then - echo " The cacerts folder of pilot is missing." - exit -1 - fi - cd $VOLUMES_ROOT/pilot/cacerts - - if [ ! -f root-key.pem ] || [ ! -f root-cert.pem ]; then - echo " The root CA certificate files of pilot are missing." - exit -1 - fi - - if [ ! -f ca-key.pem ] || [ ! -f ca-cert.pem ] || [ ! -f cert-chain.pem ]; then - echo " The CA certificate files of pilot are missing." - exit -1 - fi - - if [ ! -f gateway-key.pem ] || [ ! -f gateway-cert.pem ]; then - echo " The gateway certificate files of pilot are missing." - exit -1 - fi - - mkdir -p $VOLUMES_ROOT/pilot/config && cd "$_" - getConfig "higress-system" "configmaps" "higress-config" - checkExitCode " The ConfigMap resource of 'higress-config' doesn't exist." - fileNames=$(yq '.data | keys | .[]' <<< "$config") - if [ -z "$fileNames" ]; then - echo " Missing required files in higress-config ConfigMap." - exit -1 - fi - IFS=$'\n' - for fileName in $fileNames - do - if [ -z "$fileName" ]; then - continue - fi - echo "$config" | yq ".data.$fileName" > "./$fileName" - done -} - -checkGateway() { - echo "Checking gateway configurations..." - - if [ ! -d "$VOLUMES_ROOT/gateway/certs/" ]; then - echo " The cacerts folder of gateway is missing." - exit -1 - fi - cd $VOLUMES_ROOT/gateway/certs/ - if [ ! -f "./root-cert.pem" ] && [ ! -f "./cert-chain.pem" ] && [ ! -f "./key.pem" ]; then - echo " One or some of the certificate files of gateway is missing." - exit -1 - fi - - if [ ! -f "$VOLUMES_ROOT/gateway/podinfo/labels" ]; then - echo " The labels file of gateway are missing." - exit -1 - fi -} - -checkConsole() { - echo "Checking console configurations..." - - checkConfigExists "higress-system" "configmaps" "higress-console" - if [ $? -ne 0 ]; then - echo " The ConfigMap resource of Higress Console doesn't exist." - exit -1 - fi - checkConfigExists "higress-system" "secrets" "higress-console" - if [ $? -ne 0 ]; then - echo " The Secret resource of Higress Console doesn't exist." - exit -1 - fi -} - -checkStorage checkApiServer -checkPilot -checkGateway -checkConsole diff --git a/compose/scripts/prepare.sh b/compose/scripts/prepare.sh new file mode 100644 index 0000000..a44fa9f --- /dev/null +++ b/compose/scripts/prepare.sh @@ -0,0 +1,316 @@ +#! /bin/bash + +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +VOLUMES_ROOT="/mnt/volumes" + +API_SERVER_BASE_URL="https://apiserver:8443" + +now() { + echo "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" +} + +checkExitCode() { + # $1 message + retVal=$? + if [ $retVal -ne 0 ]; then + echo ${1:-" Command fails with $retVal"} + exit $retVal + fi +} + +checkConfigExists() { + # $1 namespace + # $2 configGroupVersion + # $3 configType: plural + # $4 configName + local namespace="$1" + local configGroupVersion="$2" + local configType="$3" + local configName="$4" + + local uriPrefix="/api" + if [[ "$configGroupVersion" == *"/"* ]]; then + uriPrefix="/apis" + fi + local url="${API_SERVER_BASE_URL}{$uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}/${configName}" + statusCode=$(curl -s -o /dev/null -w "%{http_code}" "${url}" -k) + if [ $statusCode -eq 200 ]; then + return 0 + elif [ $statusCode -eq 404 ]; then + return -1 + else + echo " Checking config ${configType}.${configName} in namespace ${namespace} failed with ${statusCode}" + exit -1 + fi +} + +getConfig() { + # $1 namespace + # $2 configGroupVersion + # $3 configType: plural + # $4 configName + local namespace="$1" + local configGroupVersion="$2" + local configType="$3" + local configName="$4" + + config="" + local uriPrefix="/api" + if [[ "$configGroupVersion" == *"/"* ]]; then + uriPrefix="/apis" + fi + local url="${API_SERVER_BASE_URL}{$uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}/${configName}" + local tmpFile=$(mktemp /tmp/higress-precheck-config.XXXXXXXXX.cfg) + local statusCode=$(curl -s -o "$tmpFile" -w "%{http_code}" "${url}" -k -H "Accept: application/yaml") + if [ $statusCode -eq 200 ]; then + config=$(cat "$tmpFile") + rm "$tmpFile" + return 0 + elif [ $statusCode -eq 404 ]; then + config="" + return -1 + else + echo ${1:-" Getting config ${configType}.${configName} in namespace ${namespace} failed with ${statusCode}"} + exit -1 + fi +} + +publishConfig() { + # $1 namespace + # $2 configGroupVersion + # $3 configType: plural + # $4 configName + # $5 content + local namespace="$1" + local configGroupVersion="$2" + local configType="$3" + local configName="$4" + local content="$5" + + local uriPrefix="/api" + if [[ "$configGroupVersion" == *"/"* ]]; then + uriPrefix="/apis" + fi + local url="${API_SERVER_BASE_URL}{$uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}" + statusCode="$(curl -s -o /dev/null -w "%{http_code}" "$url" -k -X POST -H "Content-Type: application/yaml" -d "$content")" + if [ $statusCode -ne 201 ]; then + echo " Publishing config ${configType}.${configName} to namespace ${namespace} failed with ${statusCode}" + exit -1 + fi +} + +checkStorage() { + echo "Checking config storage configurations..." + + read -r -d '' mcpbridgeContent <"./$fileName" + done +} + +checkGateway() { + echo "Checking gateway configurations..." + + if [ ! -d "$VOLUMES_ROOT/gateway/certs/" ]; then + echo " The cacerts folder of gateway is missing." + exit -1 + fi + cd $VOLUMES_ROOT/gateway/certs/ + if [ ! -f "./root-cert.pem" ] && [ ! -f "./cert-chain.pem" ] && [ ! -f "./key.pem" ]; then + echo " One or some of the certificate files of gateway is missing." + exit -1 + fi + + if [ ! -f "$VOLUMES_ROOT/gateway/podinfo/labels" ]; then + echo " The labels file of gateway are missing." + exit -1 + fi +} + +checkConsole() { + echo "Checking console configurations..." + + checkConfigExists "higress-system" "v1" "secrets" "higress-console" + if [ $? -ne 0 ]; then + echo " The ConfigMap resource \"higress-console\" doesn't exist. Create it now..." + read -r -d '' content <