From bd58388ccf086c46ba40cb855676b22f5e6687ba Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Mon, 22 Jan 2024 22:20:29 +0800 Subject: [PATCH 01/34] feat: support nacos storage Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 31 +++-- docker/Dockerfile.higress | 7 +- docker/docker.mk | 4 + docker/run-higress.sh | 29 +++++ docker/wait-for-apiserver.sh | 27 +++++ helm/core/templates/_helpers.tpl | 4 + helm/core/templates/configmap.yaml | 23 +++- .../core/templates/controller-deployment.yaml | 107 ++++++++++++++++++ helm/core/templates/controller-secret.yaml | 16 +++ helm/core/templates/controller-service.yaml | 8 +- helm/core/values.yaml | 18 +++ tools/hack/conf/nacos.yaml | 56 +++++++++ tools/hack/gen-keys.sh | 74 ++++++++++++ 13 files changed, 389 insertions(+), 15 deletions(-) create mode 100644 docker/run-higress.sh create mode 100644 docker/wait-for-apiserver.sh create mode 100644 helm/core/templates/controller-secret.yaml create mode 100644 tools/hack/conf/nacos.yaml create mode 100755 tools/hack/gen-keys.sh diff --git a/Makefile.core.mk b/Makefile.core.mk index af5c67695d..870c2b3f05 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -184,6 +184,13 @@ install-dev: pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' install-dev-wasmplugin: build-wasmplugins pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' +install-dev-nacos: pre-install + kubectl apply -f tools/hack/conf/nacos.yaml + tools/hack/gen-keys.sh + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' + +create-nacos: + kubectl apply -f tools/hack/conf/nacos.yaml uninstall: helm uninstall higress -n higress-system @@ -258,18 +265,18 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. .PHONY: kube-load-image kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) - tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 - tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 - tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 - tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest - tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 - tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 - tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 - tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 - tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 - tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest + #tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + #tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + #tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 + #tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 + #tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest + #tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 + #tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + #tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + #tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 + #tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 + #tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 + #tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest # run-higress-e2e-test starts to run ingress e2e tests. .PHONY: run-higress-e2e-test run-higress-e2e-test: diff --git a/docker/Dockerfile.higress b/docker/Dockerfile.higress index cd52b74ee5..7740e601c2 100644 --- a/docker/Dockerfile.higress +++ b/docker/Dockerfile.higress @@ -13,7 +13,12 @@ FROM ${HUB}/base:${BASE_VERSION} ARG TARGETARCH COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress +COPY wait-for-apiserver.sh /usr/local/bin/wait-for-apiserver.sh +COPY run-higress.sh /usr/local/bin/run-higress.sh + +RUN chmod +x /usr/local/bin/wait-for-apiserver.sh +RUN chmod +x /usr/local/bin/run-higress.sh USER 1337:1337 -ENTRYPOINT ["/usr/local/bin/higress"] +ENTRYPOINT ["/usr/local/bin/run-higress.sh"] diff --git a/docker/docker.mk b/docker/docker.mk index aa01abdfc0..d7fa7a762c 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -14,12 +14,16 @@ docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} docker.higress: $(OUT_LINUX)/higress +docker.higress: docker/wait-for-apiserver.sh +docker.higress: docker/run-higress.sh docker.higress: docker/Dockerfile.higress $(HIGRESS_DOCKER_RULE) docker.higress-buildx: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} docker.higress-buildx: $(AMD64_OUT_LINUX)/higress docker.higress-buildx: $(ARM64_OUT_LINUX)/higress +docker.higress: docker/wait-for-apiserver.sh +docker.higress: docker/run-higress.sh docker.higress-buildx: docker/Dockerfile.higress $(HIGRESS_DOCKER_BUILDX_RULE) diff --git a/docker/run-higress.sh b/docker/run-higress.sh new file mode 100644 index 0000000000..242bc87cd7 --- /dev/null +++ b/docker/run-higress.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http:www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +args=("$@") + +for arg in "${args[@]}"; do + echo "Argument: $arg" +done + +/usr/local/bin/wait-for-apiserver.sh + +echo "starting higress..." + +/usr/local/bin/higress "$@" + +echo "higress started." \ No newline at end of file diff --git a/docker/wait-for-apiserver.sh b/docker/wait-for-apiserver.sh new file mode 100644 index 0000000000..d08ca4e84c --- /dev/null +++ b/docker/wait-for-apiserver.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http:www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# if has HIGRESS_APISERVER_SVC env, wait for apiserver ready +if [ -n "$HIGRESS_APISERVER_SVC" ]; then + while true; do + echo "testing higress apiserver is ready to connect..." + nc -z "$HIGRESS_APISERVER_SVC" "${HIGRESS_APISERVER_PORT}" + if [ $? -eq 0 ]; then + break + fi + sleep 1 + done +fi \ No newline at end of file diff --git a/helm/core/templates/_helpers.tpl b/helm/core/templates/_helpers.tpl index 392fd04ef4..103dfe7089 100644 --- a/helm/core/templates/_helpers.tpl +++ b/helm/core/templates/_helpers.tpl @@ -101,3 +101,7 @@ higress: {{ include "controller.name" . }} true {{- end }} {{- end }} + +{{- define "apiserver.name" -}} +{{- .Values.apiserver.name | default "higress-apiserver" -}} +{{- end }} \ No newline at end of file diff --git a/helm/core/templates/configmap.yaml b/helm/core/templates/configmap.yaml index 6ef7b87bf4..771541badf 100644 --- a/helm/core/templates/configmap.yaml +++ b/helm/core/templates/configmap.yaml @@ -105,7 +105,28 @@ metadata: labels: {{- include "gateway.labels" . | nindent 4 }} data: - + {{- if .Values.apiserver.enabled }} + kubeconfig: | + apiVersion: v1 + kind: Config + clusters: + - name: {{ .Release.Namespace }} + cluster: + server: https://{{ .Values.apiserver.addr }}:{{ .Values.apiserver.securePort }} + insecure-skip-tls-verify: true + users: + - name: higress-admin + user: + client-certificate-data: {{ .Files.Get "files/api/client.crt" | b64enc }} + client-key-data: {{ .Files.Get "files/api/client.key" | b64enc }} + contexts: + - name: higress + context: + cluster: higress + user: higress-admin + preferences: {} + current-context: higress + {{- end }} # Configuration file for the mesh networks to be used by the Split Horizon EDS. meshNetworks: |- {{- if .Values.global.meshNetworks }} diff --git a/helm/core/templates/controller-deployment.yaml b/helm/core/templates/controller-deployment.yaml index f561b182ac..dd5b3ba056 100644 --- a/helm/core/templates/controller-deployment.yaml +++ b/helm/core/templates/controller-deployment.yaml @@ -189,6 +189,9 @@ spec: image: "{{ .Values.controller.hub | default .Values.global.hub }}/{{ .Values.controller.image | default "higress" }}:{{ .Values.controller.tag | default .Chart.AppVersion }}" args: - "serve" + {{- if .Values.apiserver.enabled }} + - --kubeconfig=/etc/istio/config/kubeconfig + {{- end }} - --gatewaySelectorKey=higress - --gatewaySelectorValue={{ .Release.Namespace }}-{{ include "gateway.name" . }} {{- if not .Values.global.enableStatus }} @@ -199,6 +202,12 @@ spec: - --watchNamespace={{ .Values.global.watchNamespace }} {{- end }} env: + {{- if .Values.apiserver.enabled }} + - name: HIGRESS_APISERVER_SVC + value: {{ .Values.apiserver.addr | quote }} + - name: HIGRESS_APISERVER_PORT + value: {{ .Values.apiserver.securePort | quote }} + {{- end }} - name: POD_NAME valueFrom: fieldRef: @@ -237,6 +246,96 @@ spec: volumeMounts: - name: log mountPath: /var/log + {{- if .Values.apiserver.enabled }} + - name: config + mountPath: /etc/istio/config + {{- end }} + {{- if .Values.apiserver.enabled }} + - name: {{ .Values.apiserver.name }} + image: "{{ .Values.apiserver.hub | default .Values.global.hub }}/{{ .Values.apiserver.image }}:{{ .Values.apiserver.tag }}" + imagePullPolicy: {{ .Values.apiserver.imagePullPolicy }} + args: + - --secure-port + - {{ .Values.apiserver.securePort | quote }} + - --client-ca-file + - /etc/api/ca.crt + - --tls-cert-file + - /etc/api/server.crt + - --tls-private-key-file + - /etc/api/server.key + - --storage + - {{ .Values.apiserver.storage }} + - --{{ .Values.apiserver.storage }}-server + - {{ .Values.apiserver.serverAddr }} + - --{{ .Values.apiserver.storage }}-username + - {{ .Values.apiserver.username }} + - --{{ .Values.apiserver.storage }}-password + - {{ .Values.apiserver.password }} + - --{{ .Values.apiserver.storage }}-ns-id + - {{ .Values.apiserver.namespaceID }} + - --{{ .Values.apiserver.storage }}-encryption-key-file + - /etc/api/nacos.key + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + ports: + - containerPort: 8443 + hostPort: 8443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 30 + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 3 + resources: { } + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1337 + runAsNonRoot: true + runAsUser: 1337 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/api + name: cert-config + readOnly: true + - mountPath: /tmp/nacos + name: nacos-data + {{- end }} {{- with .Values.controller.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -252,6 +351,14 @@ spec: volumes: - name: log emptyDir: {} + {{- if .Values.apiserver.enabled }} + - name: {{ .Values.apiserver.storage }}-data + emptyDir: {} + - name: cert-config + secret: + defaultMode: 420 + secretName: higress-apiserver + {{- end }} {{- if not .Values.global.enableHigressIstio }} - name: config configMap: diff --git a/helm/core/templates/controller-secret.yaml b/helm/core/templates/controller-secret.yaml new file mode 100644 index 0000000000..40f2f39e8f --- /dev/null +++ b/helm/core/templates/controller-secret.yaml @@ -0,0 +1,16 @@ +{{- if .Values.apiserver.enabled }} +apiVersion: v1 +kind: Secret +metadata: + annotations: + helm.sh/hook: pre-install,pre-upgrade + name: {{ include "apiserver.name" . }} + namespace: {{ .Release.Namespace }} +data: + ca.crt: {{ .Files.Get "files/api/ca.crt" | b64enc }} + ca.key: {{ .Files.Get "files/api/ca.key" | b64enc }} + server.crt: {{ .Files.Get "files/api/server.crt" | b64enc }} + server.key: {{ .Files.Get "files/api/server.key" | b64enc }} + client.key: {{ .Files.Get "files/api/client.crt" | b64enc }} + nacos.key: {{ .Files.Get "files/api/nacos.key" | b64enc }} +{{- end }} \ No newline at end of file diff --git a/helm/core/templates/controller-service.yaml b/helm/core/templates/controller-service.yaml index 70b22351e4..fd5376e7bd 100644 --- a/helm/core/templates/controller-service.yaml +++ b/helm/core/templates/controller-service.yaml @@ -23,6 +23,12 @@ spec: - port: 15014 name: http-monitoring # prometheus stats protocol: TCP + {{- if .Values.apiserver.enabled }} + - port: {{ .Values.apiserver.securePort }} + name: https-apiserver # mTLS with k8s-signed cert + protocol: TCP + targetPort: {{ .Values.apiserver.securePort }} + {{- end }} {{- end }} selector: - {{- include "controller.selectorLabels" . | nindent 4 }} + {{- include "controller.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/helm/core/values.yaml b/helm/core/values.yaml index 7f3accb82f..c5c6d643bb 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -615,6 +615,24 @@ pilot: # Additional labels to apply on the pod level for monitoring and logging configuration. podLabels: {} +## Higress ApiServer Storage Settings +## Todo simplify this +apiserver: + addr: 127.0.0.1 + enabled: false + name: "higress-apiserver" + hub: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress + tag: 0.0.10 + image: api-server + imagePullPolicy: IfNotPresent + securePort: 8443 + storage: nacos + serverAddr: http://127.0.0.1:8848 + username: "" + password: "" + namespaceID: "" + rsaKeyLength: 4096 + serviceName: "higress-controller" # Skywalking config settings skywalking: diff --git a/tools/hack/conf/nacos.yaml b/tools/hack/conf/nacos.yaml new file mode 100644 index 0000000000..5c8fd61603 --- /dev/null +++ b/tools/hack/conf/nacos.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains the base resources that most conformance tests will rely +# on. This includes 3 namespaces along with Gateways, Services and Deployments +# that can be used as backends for routing traffic. The most important +# resources included are the Gateways (all in the higress-conformance-infra +# namespace): +# - same-namespace (only supports route in same ns) +# - all-namespaces (supports routes in all ns) +# - backend-namespaces (supports routes in ns with backend label) +apiVersion: v1 +kind: Namespace +metadata: + name: higress-system +--- +apiVersion: v1 +kind: Pod +metadata: + name: nacos + namespace: higress-system + labels: + name: nacos +spec: + containers: + - name: nacos + image: swsk33/nacos-standalone:2.2.3 + ports: + - containerPort: 8848 + - containerPort: 9848 +--- +apiVersion: v1 +kind: Service +metadata: + name: nacos-service + namespace: higress-system +spec: + selector: + name: nacos + clusterIP: None + ports: + - name: foo # name is not required for single-port Services + port: 8848 + - name: bar + port: 9848 diff --git a/tools/hack/gen-keys.sh b/tools/hack/gen-keys.sh new file mode 100755 index 0000000000..c518257f20 --- /dev/null +++ b/tools/hack/gen-keys.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http:www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RSA_KEY_LENGTH="${RSA_KEY_LENGTH:-4096}" + +VOLUMES_ROOT="helm/core/files" + +initializeApiServer() { + echo "Initializing API server configurations..." + mkdir -p "$VOLUMES_ROOT/api" && cd "$_" + checkExitCode "Creating volume for API server fails with $?" + + if [ ! -f ca.key ] || [ ! -f ca.crt ]; then + echo " Generating CA certificate..."; + openssl req -nodes -new -x509 -days 36500 -keyout ca.key -out ca.crt -subj "/CN=higress-root-ca/O=higress" > /dev/null 2>&1 + checkExitCode " Generating CA certificate for API server fails with $?"; + else + echo " CA certificate already exists."; + fi + + if [ ! -f server.key ] || [ ! -f server.crt ]; then + echo " Generating server certificate..." + openssl req -out server.csr -new -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout server.key -subj "/CN=higress-api-server/O=higress" > /dev/null 2>&1 \ + && openssl x509 -req -days 36500 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -sha256 -out server.crt > /dev/null 2>&1 + checkExitCode " Generating server certificate fails with $?"; + else + echo " Server certificate already exists."; + fi + + if [ ! -f nacos.key ]; then + echo " Generating data encryption key..." + if [ -z "$NACOS_DATA_ENC_KEY" ]; then + cat /dev/urandom | tr -dc '[:graph:]' | head -c 32 > nacos.key + else + echo -n "$NACOS_DATA_ENC_KEY" > nacos.key + fi + else + echo " Client certificate already exists."; + fi + + if [ ! -f client.key ] || [ ! -f client.crt ]; then + echo " Generating client certificate..." + openssl req -out client.csr -new -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout client.key -subj "/CN=higress/O=system:masters" > /dev/null 2>&1 \ + && openssl x509 -req -days 36500 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 02 -sha256 -out client.crt > /dev/null 2>&1 + checkExitCode " Generating client certificate fails with $?"; + else + echo " Client certificate already exists."; + fi + echo " Done." +} + +checkExitCode() { + # $1 message + retVal=$? + if [ $retVal -ne 0 ]; then + echo ${1:-" Command fails with $retVal"} + exit $retVal + fi +} + +initializeApiServer \ No newline at end of file From 625c61de879e9183150335483691e73148dd5a51 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Mon, 22 Jan 2024 22:24:34 +0800 Subject: [PATCH 02/34] recover something Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 870c2b3f05..e456c6dd6a 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -265,18 +265,18 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. .PHONY: kube-load-image kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) - #tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - #tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 - #tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 - #tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 - #tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest - #tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 - #tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - #tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 - #tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 - #tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 - #tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 - #tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest + tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 + tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 + tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest + tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 + tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 + tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 + tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 + tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest # run-higress-e2e-test starts to run ingress e2e tests. .PHONY: run-higress-e2e-test run-higress-e2e-test: From d848bb89ea4d3d8ed8bf4e8d6bdb45f6ae53fd40 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Mon, 22 Jan 2024 22:45:16 +0800 Subject: [PATCH 03/34] recover something Signed-off-by: sjcsjc123 <1401189096@qq.com> --- helm/core/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/core/values.yaml b/helm/core/values.yaml index c5c6d643bb..80850cc75d 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -632,7 +632,6 @@ apiserver: password: "" namespaceID: "" rsaKeyLength: 4096 - serviceName: "higress-controller" # Skywalking config settings skywalking: From e8888cbb1f7dad96226bf434e8b5536e91575081 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Mon, 22 Jan 2024 22:46:28 +0800 Subject: [PATCH 04/34] recover something Signed-off-by: sjcsjc123 <1401189096@qq.com> --- helm/core/templates/controller-service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/core/templates/controller-service.yaml b/helm/core/templates/controller-service.yaml index fd5376e7bd..938d7cf1f3 100644 --- a/helm/core/templates/controller-service.yaml +++ b/helm/core/templates/controller-service.yaml @@ -25,7 +25,7 @@ spec: protocol: TCP {{- if .Values.apiserver.enabled }} - port: {{ .Values.apiserver.securePort }} - name: https-apiserver # mTLS with k8s-signed cert + name: https-apiserver protocol: TCP targetPort: {{ .Values.apiserver.securePort }} {{- end }} From d01a220244075b1a527ba8b2a6bfda2c53476c97 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Mon, 22 Jan 2024 22:53:52 +0800 Subject: [PATCH 05/34] simple Signed-off-by: sjcsjc123 <1401189096@qq.com> --- docker/Dockerfile.higress | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile.higress b/docker/Dockerfile.higress index 7740e601c2..3527dd8884 100644 --- a/docker/Dockerfile.higress +++ b/docker/Dockerfile.higress @@ -16,8 +16,7 @@ COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress COPY wait-for-apiserver.sh /usr/local/bin/wait-for-apiserver.sh COPY run-higress.sh /usr/local/bin/run-higress.sh -RUN chmod +x /usr/local/bin/wait-for-apiserver.sh -RUN chmod +x /usr/local/bin/run-higress.sh +RUN chmod +x /usr/local/bin/*.sh USER 1337:1337 From d8bb8d3a5fcec293461f1f933a6b9198c19ba6a9 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Tue, 23 Jan 2024 12:56:48 +0800 Subject: [PATCH 06/34] fix bug Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 5 +---- helm/core/templates/configmap.yaml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index e456c6dd6a..c54ef35586 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -187,10 +187,7 @@ install-dev-wasmplugin: build-wasmplugins pre-install install-dev-nacos: pre-install kubectl apply -f tools/hack/conf/nacos.yaml tools/hack/gen-keys.sh - helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' - -create-nacos: - kubectl apply -f tools/hack/conf/nacos.yaml + helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' uninstall: helm uninstall higress -n higress-system diff --git a/helm/core/templates/configmap.yaml b/helm/core/templates/configmap.yaml index 771541badf..073bdeff19 100644 --- a/helm/core/templates/configmap.yaml +++ b/helm/core/templates/configmap.yaml @@ -110,7 +110,7 @@ data: apiVersion: v1 kind: Config clusters: - - name: {{ .Release.Namespace }} + - name: {{ .Values.clusterName | default "higress" }} cluster: server: https://{{ .Values.apiserver.addr }}:{{ .Values.apiserver.securePort }} insecure-skip-tls-verify: true From 5aed7f412bd2197acdfe6e37ccc69b339a719591 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Tue, 23 Jan 2024 14:16:27 +0800 Subject: [PATCH 07/34] modify nacos image Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/base/nacos.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e/conformance/base/nacos.yaml b/test/e2e/conformance/base/nacos.yaml index 459da47d40..c2af14e536 100644 --- a/test/e2e/conformance/base/nacos.yaml +++ b/test/e2e/conformance/base/nacos.yaml @@ -31,9 +31,10 @@ metadata: spec: containers: - name: nacos-standlone-rc3 - image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 + image: swsk33/nacos-standalone:2.2.3 ports: - containerPort: 8848 + - containerPort: 9848 --- apiVersion: v1 kind: Service @@ -47,3 +48,5 @@ spec: ports: - name: foo # name is not required for single-port Services port: 8848 + - name: bar + port: 9848 From 0ffcc30922e4f597f3fb288dba5651daa6e62407 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Tue, 23 Jan 2024 14:20:05 +0800 Subject: [PATCH 08/34] update nacos image in Makefile Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index c54ef35586..0c65a467c5 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -263,13 +263,13 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/docker-pull-image.sh docker.io/swsk33/nacos-standalone 2.2.3 tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.3 tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 From c0c4e38b96376b9a9ec01fa5dc43595f9b7ea12e Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Tue, 23 Jan 2024 16:39:04 +0800 Subject: [PATCH 09/34] add nacos client Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 +- helm/core/templates/configmap.yaml | 23 +----- helm/core/templates/controller-secret.yaml | 16 ----- test/e2e/conformance/utils/nacos/nacos.go | 59 +++++++++++++++ .../{gen-keys.sh => gen-secret-configmap.sh} | 71 +++++++++++++++++-- 5 files changed, 127 insertions(+), 44 deletions(-) delete mode 100644 helm/core/templates/controller-secret.yaml create mode 100644 test/e2e/conformance/utils/nacos/nacos.go rename tools/hack/{gen-keys.sh => gen-secret-configmap.sh} (57%) diff --git a/Makefile.core.mk b/Makefile.core.mk index 0c65a467c5..1fb120b677 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -269,7 +269,7 @@ kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster us tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.3 + tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.302 tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 diff --git a/helm/core/templates/configmap.yaml b/helm/core/templates/configmap.yaml index 073bdeff19..6ef7b87bf4 100644 --- a/helm/core/templates/configmap.yaml +++ b/helm/core/templates/configmap.yaml @@ -105,28 +105,7 @@ metadata: labels: {{- include "gateway.labels" . | nindent 4 }} data: - {{- if .Values.apiserver.enabled }} - kubeconfig: | - apiVersion: v1 - kind: Config - clusters: - - name: {{ .Values.clusterName | default "higress" }} - cluster: - server: https://{{ .Values.apiserver.addr }}:{{ .Values.apiserver.securePort }} - insecure-skip-tls-verify: true - users: - - name: higress-admin - user: - client-certificate-data: {{ .Files.Get "files/api/client.crt" | b64enc }} - client-key-data: {{ .Files.Get "files/api/client.key" | b64enc }} - contexts: - - name: higress - context: - cluster: higress - user: higress-admin - preferences: {} - current-context: higress - {{- end }} + # Configuration file for the mesh networks to be used by the Split Horizon EDS. meshNetworks: |- {{- if .Values.global.meshNetworks }} diff --git a/helm/core/templates/controller-secret.yaml b/helm/core/templates/controller-secret.yaml deleted file mode 100644 index 40f2f39e8f..0000000000 --- a/helm/core/templates/controller-secret.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if .Values.apiserver.enabled }} -apiVersion: v1 -kind: Secret -metadata: - annotations: - helm.sh/hook: pre-install,pre-upgrade - name: {{ include "apiserver.name" . }} - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ .Files.Get "files/api/ca.crt" | b64enc }} - ca.key: {{ .Files.Get "files/api/ca.key" | b64enc }} - server.crt: {{ .Files.Get "files/api/server.crt" | b64enc }} - server.key: {{ .Files.Get "files/api/server.key" | b64enc }} - client.key: {{ .Files.Get "files/api/client.crt" | b64enc }} - nacos.key: {{ .Files.Get "files/api/nacos.key" | b64enc }} -{{- end }} \ No newline at end of file diff --git a/test/e2e/conformance/utils/nacos/nacos.go b/test/e2e/conformance/utils/nacos/nacos.go new file mode 100644 index 0000000000..16fab79a70 --- /dev/null +++ b/test/e2e/conformance/utils/nacos/nacos.go @@ -0,0 +1,59 @@ +package nacos + +import ( + "net/url" + "strconv" + "strings" + + "github.com/nacos-group/nacos-sdk-go/v2/clients" + "github.com/nacos-group/nacos-sdk-go/v2/clients/config_client" + "github.com/nacos-group/nacos-sdk-go/v2/common/constant" + "github.com/nacos-group/nacos-sdk-go/v2/vo" +) + +func NewClient(addr string) (config_client.IConfigClient, error) { + cc := constant.NewClientConfig( + constant.WithNamespaceId(""), + constant.WithUsername(""), + constant.WithPassword(""), + constant.WithLogLevel("info"), + ) + + serverUrl, err := url.Parse(addr) + if err != nil { + return nil, err + } + rawPort := serverUrl.Port() + var port uint64 + if rawPort != "" { + port, err = strconv.ParseUint(rawPort, 10, 0) + if err != nil || port < 1 || port > 65535 { + return nil, err + } + } else { + port = 80 + } + path := serverUrl.Path + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } + sc := []constant.ServerConfig{ + { + IpAddr: serverUrl.Hostname(), + ContextPath: path, + Port: port, + Scheme: serverUrl.Scheme, + }, + } + + client, err := clients.NewConfigClient( + vo.NacosClientParam{ + ClientConfig: cc, + ServerConfigs: sc, + }, + ) + if err != nil { + return nil, err + } + return client, nil +} diff --git a/tools/hack/gen-keys.sh b/tools/hack/gen-secret-configmap.sh similarity index 57% rename from tools/hack/gen-keys.sh rename to tools/hack/gen-secret-configmap.sh index c518257f20..70cd2cd7c8 100755 --- a/tools/hack/gen-keys.sh +++ b/tools/hack/gen-secret-configmap.sh @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -RSA_KEY_LENGTH="${RSA_KEY_LENGTH:-4096}" - -VOLUMES_ROOT="helm/core/files" +VOLUMES_ROOT="/tmp/higress-apiserver" +RSA_KEY_LENGTH=4096 +NAMESPACE="${NAMESPACE:-higress-system}" +CLUSTER_NAME="${CLUSTER_NAME:-higress}" +APISERVER_ADDRESS="${APISERVER_ADDRESS:-https://127.0.0.1:8443}" initializeApiServer() { echo "Initializing API server configurations..." + mkdir -p "$VOLUMES_ROOT/api" && cd "$_" checkExitCode "Creating volume for API server fails with $?" @@ -59,7 +62,64 @@ initializeApiServer() { else echo " Client certificate already exists."; fi - echo " Done." +} + +applySecretConfigmap() { + # create namespace if not exists + kubectl get namespace $NAMESPACE > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Creating namespace $NAMESPACE..." + kubectl create namespace $NAMESPACE + checkExitCode "Creating namespace fails with $?" + fi + + echo "Applying secret $NAMESPACE/higress-apiserver..." + kubectl apply -f - < Date: Tue, 23 Jan 2024 17:51:32 +0800 Subject: [PATCH 10/34] add ci for nacos storage Signed-off-by: sjcsjc123 <1401189096@qq.com> --- .github/workflows/build-and-test.yaml | 39 +++++++- Makefile.core.mk | 19 +++- .../utils/configcenter/nacos/nacos.go | 97 +++++++++++++++++++ .../conformance/utils/configcenter/storage.go | 20 ++++ test/e2e/conformance/utils/flags/flags.go | 3 + .../e2e/conformance/utils/kubernetes/apply.go | 35 +++++++ test/e2e/conformance/utils/nacos/nacos.go | 59 ----------- test/e2e/conformance/utils/suite/suite.go | 34 ++++++- test/e2e/e2e_test.go | 2 + 9 files changed, 243 insertions(+), 65 deletions(-) create mode 100644 test/e2e/conformance/utils/configcenter/nacos/nacos.go create mode 100644 test/e2e/conformance/utils/configcenter/storage.go delete mode 100644 test/e2e/conformance/utils/nacos/nacos.go diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 149c7cf52e..a8faf2e648 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -134,7 +134,44 @@ jobs: - name: "Run Higress E2E Conformance Tests" run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test - + + + + + higress-conformance-test-nacos: + runs-on: ubuntu-latest + needs: [ build ] + steps: + - uses: actions/checkout@v3 + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Setup Golang Caches + uses: actions/cache@v3 + with: + path: |- + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-go + + - name: Setup Submodule Caches + uses: actions/cache@v3 + with: + path: |- + .git/modules + key: ${{ runner.os }}-submodules-cache-${{ github.run_id }} + restore-keys: ${{ runner.os }}-submodules-cache + + - run: git stash # restore patch + + - name: "Run Higress E2E Conformance Nacos Tests" + run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test-nacos + publish: runs-on: ubuntu-latest needs: [higress-conformance-test,gateway-conformance-test] diff --git a/Makefile.core.mk b/Makefile.core.mk index 1fb120b677..be0623cdb0 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -245,6 +245,14 @@ higress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-bui .PHONY: higress-wasmplugin-test higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-higress-e2e-test-wasmplugin delete-cluster +# higress-conformance-test-nacos runs ingress nacos tests. +.PHONY: higress-conformance-test-nacos +higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward run-higress-e2e-test-nacos delete-cluster + +.PHONY: port-forward +port-forward: + kubectl port-forward -n higress-system svc/nacos-service 8848:8848 & + # create-cluster creates a kube cluster with kind. .PHONY: create-cluster create-cluster: $(tools/kind) @@ -269,7 +277,7 @@ kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster us tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.302 + tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.3 tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 @@ -293,3 +301,12 @@ run-higress-e2e-test-wasmplugin: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true + +.PHONY: run-higress-e2e-test-nacos +run-higress-e2e-test-nacos: + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available + @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available + go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --enableApiServer=true \ No newline at end of file diff --git a/test/e2e/conformance/utils/configcenter/nacos/nacos.go b/test/e2e/conformance/utils/configcenter/nacos/nacos.go new file mode 100644 index 0000000000..4ffa5e6fbc --- /dev/null +++ b/test/e2e/conformance/utils/configcenter/nacos/nacos.go @@ -0,0 +1,97 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nacos + +import ( + "net/url" + "strconv" + "strings" + + cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" + "github.com/nacos-group/nacos-sdk-go/v2/clients" + "github.com/nacos-group/nacos-sdk-go/v2/clients/config_client" + "github.com/nacos-group/nacos-sdk-go/v2/common/constant" + "github.com/nacos-group/nacos-sdk-go/v2/vo" +) + +type storage struct { + client config_client.IConfigClient +} + +func NewClient(addr string) (cc.Storage, error) { + clientConfig := constant.NewClientConfig( + constant.WithNamespaceId(""), + constant.WithUsername(""), + constant.WithPassword(""), + constant.WithLogLevel("info"), + ) + + serverUrl, err := url.Parse(addr) + if err != nil { + return nil, err + } + rawPort := serverUrl.Port() + var port uint64 + if rawPort != "" { + port, err = strconv.ParseUint(rawPort, 10, 0) + if err != nil || port < 1 || port > 65535 { + return nil, err + } + } else { + port = 80 + } + path := serverUrl.Path + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } + serverConfigs := []constant.ServerConfig{ + { + IpAddr: serverUrl.Hostname(), + ContextPath: path, + Port: port, + Scheme: serverUrl.Scheme, + }, + } + + client, err := clients.NewConfigClient( + vo.NacosClientParam{ + ClientConfig: clientConfig, + ServerConfigs: serverConfigs, + }, + ) + if err != nil { + return nil, err + } + return storage{ + client: client, + }, nil +} + +func (s storage) PublishConfig(dataId, group, content string) error { + _, err := s.client.PublishConfig(vo.ConfigParam{ + DataId: dataId, + Group: group, + Content: content, + }) + return err +} + +func (s storage) DeleteConfig(dataId, group string) error { + _, err := s.client.DeleteConfig(vo.ConfigParam{ + DataId: dataId, + Group: group, + }) + return err +} diff --git a/test/e2e/conformance/utils/configcenter/storage.go b/test/e2e/conformance/utils/configcenter/storage.go new file mode 100644 index 0000000000..d5c9e3bfab --- /dev/null +++ b/test/e2e/conformance/utils/configcenter/storage.go @@ -0,0 +1,20 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +type Storage interface { + PublishConfig(dataId, group, content string) error + DeleteConfig(dataId, group string) error +} diff --git a/test/e2e/conformance/utils/flags/flags.go b/test/e2e/conformance/utils/flags/flags.go index 29b3c342cf..e70485a10a 100644 --- a/test/e2e/conformance/utils/flags/flags.go +++ b/test/e2e/conformance/utils/flags/flags.go @@ -27,4 +27,7 @@ var ( WasmPluginType = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP") WasmPluginName = flag.String("wasmPluginName", "", "Define wasm plugin name") IsEnvoyConfigTest = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test") + EnableApiServer = flag.Bool("enableApiServer", false, "Determine if enable api server") + // TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace + Storage = flag.String("storage", "http://127.0.0.1:8848", "Define storage") ) diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index 28a44ff0e2..4861ce102f 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -24,6 +24,7 @@ import ( "testing" ingress "github.com/alibaba/higress/test/e2e/conformance" + cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -249,3 +250,37 @@ func getContentsFromPathOrURL(location string, timeoutConfig config.TimeoutConfi } return bytes.NewBuffer(b), nil } + +// MustPublishConfig publish config to config center +func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutConfig, location string, cleanup bool, cc cc.Storage) { + data, err := getContentsFromPathOrURL(location, timeoutConfig) + require.NoError(t, err) + + decoder := yaml.NewYAMLOrJSONDecoder(data, 4096) + + resources, err := a.prepareResources(t, decoder) + if err != nil { + t.Logf("🧳 Manifest: %s", data.String()) + require.NoErrorf(t, err, "error parsing manifest") + } + + for _, r := range resources { + // TODO: Maybe add s or es + dataId := r.GetKind() + "es" + group := r.GetNamespace() + // convert r to string + var content []byte + content, err = r.MarshalJSON() + require.NoError(t, err) + // publish + err = cc.PublishConfig(dataId, group, string(content)) + require.NoError(t, err) + if cleanup { + t.Cleanup(func() { + // delete + err = cc.DeleteConfig(dataId, group) + require.NoError(t, err) + }) + } + } +} diff --git a/test/e2e/conformance/utils/nacos/nacos.go b/test/e2e/conformance/utils/nacos/nacos.go deleted file mode 100644 index 16fab79a70..0000000000 --- a/test/e2e/conformance/utils/nacos/nacos.go +++ /dev/null @@ -1,59 +0,0 @@ -package nacos - -import ( - "net/url" - "strconv" - "strings" - - "github.com/nacos-group/nacos-sdk-go/v2/clients" - "github.com/nacos-group/nacos-sdk-go/v2/clients/config_client" - "github.com/nacos-group/nacos-sdk-go/v2/common/constant" - "github.com/nacos-group/nacos-sdk-go/v2/vo" -) - -func NewClient(addr string) (config_client.IConfigClient, error) { - cc := constant.NewClientConfig( - constant.WithNamespaceId(""), - constant.WithUsername(""), - constant.WithPassword(""), - constant.WithLogLevel("info"), - ) - - serverUrl, err := url.Parse(addr) - if err != nil { - return nil, err - } - rawPort := serverUrl.Port() - var port uint64 - if rawPort != "" { - port, err = strconv.ParseUint(rawPort, 10, 0) - if err != nil || port < 1 || port > 65535 { - return nil, err - } - } else { - port = 80 - } - path := serverUrl.Path - if strings.HasSuffix(path, "/") { - path = path[:len(path)-1] - } - sc := []constant.ServerConfig{ - { - IpAddr: serverUrl.Hostname(), - ContextPath: path, - Port: port, - Scheme: serverUrl.Scheme, - }, - } - - client, err := clients.NewConfigClient( - vo.NacosClientParam{ - ClientConfig: cc, - ServerConfigs: sc, - }, - ) - if err != nil { - return nil, err - } - return client, nil -} diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 2f6969ba86..2b9ab048e5 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -15,11 +15,14 @@ package suite import ( "fmt" + "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" "testing" "github.com/alibaba/higress/test/e2e/conformance/utils/config" + cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper" + "github.com/stretchr/testify/require" "istio.io/istio/pilot/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -38,6 +41,9 @@ type ConformanceTestSuite struct { SkipTests sets.Set TimeoutConfig config.TimeoutConfig SupportedFeatures sets.Set + EnableApiServer bool + Storage string + ConfigCenter cc.Storage } // Options can be used to initialize a ConformanceTestSuite. @@ -63,6 +69,9 @@ type Options struct { // IsEnvoyConfigTest indicates whether or not the test is for envoy config IsEnvoyConfigTest bool + + EnableApiServer bool + Storage string } type WASMOptions struct { @@ -112,7 +121,9 @@ func New(s Options) *ConformanceTestSuite { Applier: kubernetes.Applier{ NamespaceLabels: s.NamespaceLabels, }, - TimeoutConfig: s.TimeoutConfig, + TimeoutConfig: s.TimeoutConfig, + EnableApiServer: s.EnableApiServer, + Storage: s.Storage, } // apply defaults @@ -132,6 +143,14 @@ func New(s Options) *ConformanceTestSuite { // Setup ensures the base resources required for conformance tests are installed // in the cluster. It also ensures that all relevant resources are ready. func (suite *ConformanceTestSuite) Setup(t *testing.T) { + if suite.EnableApiServer { + t.Logf("📦 Test Setup: Ensuring ApiServer Storage has been accepted") + configClient, err := nacos.NewClient(suite.Storage) + require.NoError(t, err) + require.NotNil(t, configClient) + suite.ConfigCenter = configClient + } + t.Logf("📦 Test Setup: Ensuring IngressClass has been accepted") suite.Applier.IngressClass = suite.IngressClassName @@ -220,9 +239,16 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, manifestLocation) } - for _, manifestLocation := range test.Manifests { - t.Logf("🧳 Applying Manifests: %s", manifestLocation) - suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup) + if suite.EnableApiServer { + for _, manifestLocation := range test.Manifests { + t.Logf("🧳 Applying ApiServer Storage Manifests") + suite.Applier.MustPublishConfig(t, suite.TimeoutConfig, manifestLocation, !test.NotCleanup, suite.ConfigCenter) + } + } else { + for _, manifestLocation := range test.Manifests { + t.Logf("🧳 Applying Manifests: %s", manifestLocation) + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup) + } } test.Test(t, suite) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e32a6ec0c5..c1633b5fbd 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -51,6 +51,8 @@ func TestHigressConformanceTests(t *testing.T) { }, GatewayAddress: "localhost", EnableAllSupportedFeatures: true, + EnableApiServer: *flags.EnableApiServer, + Storage: *flags.Storage, IsEnvoyConfigTest: *flags.IsEnvoyConfigTest, }) From 9bb4920f2293f30addfb86b4a9e801788ddc5cb0 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Tue, 23 Jan 2024 17:53:38 +0800 Subject: [PATCH 11/34] delete something Signed-off-by: sjcsjc123 <1401189096@qq.com> --- .github/workflows/build-and-test.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index a8faf2e648..8b603522c5 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -135,9 +135,6 @@ jobs: - name: "Run Higress E2E Conformance Tests" run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test - - - higress-conformance-test-nacos: runs-on: ubuntu-latest needs: [ build ] From 31ca38548eeb8f05f28753043d37358b144e768f Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 00:25:19 +0800 Subject: [PATCH 12/34] fix bug with start fail Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 4 ++-- helm/core/templates/controller-deployment.yaml | 10 ++++++++++ tools/hack/gen-secret-configmap.sh | 7 ++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index be0623cdb0..bba2a65104 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -186,8 +186,8 @@ install-dev-wasmplugin: build-wasmplugins pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' install-dev-nacos: pre-install kubectl apply -f tools/hack/conf/nacos.yaml - tools/hack/gen-keys.sh - helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' + tools/hack/gen-secret-configmap.sh + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' uninstall: helm uninstall higress -n higress-system diff --git a/helm/core/templates/controller-deployment.yaml b/helm/core/templates/controller-deployment.yaml index dd5b3ba056..f6d3f1a25f 100644 --- a/helm/core/templates/controller-deployment.yaml +++ b/helm/core/templates/controller-deployment.yaml @@ -361,8 +361,18 @@ spec: {{- end }} {{- if not .Values.global.enableHigressIstio }} - name: config + {{- if not .Values.apiserver.enabled }} configMap: name: higress-config + {{- end }} + {{- if .Values.apiserver.enabled }} + projected: + sources: + - configMap: + name: higress-config + - configMap: + name: higress-apiserver + {{- end }} # Technically not needed on this pod - but it helps debugging/testing SDS # Should be removed after everything works. - emptyDir: diff --git a/tools/hack/gen-secret-configmap.sh b/tools/hack/gen-secret-configmap.sh index 70cd2cd7c8..ea95e7ac18 100755 --- a/tools/hack/gen-secret-configmap.sh +++ b/tools/hack/gen-secret-configmap.sh @@ -19,6 +19,7 @@ RSA_KEY_LENGTH=4096 NAMESPACE="${NAMESPACE:-higress-system}" CLUSTER_NAME="${CLUSTER_NAME:-higress}" APISERVER_ADDRESS="${APISERVER_ADDRESS:-https://127.0.0.1:8443}" +SECRET_NAME="${SECRET_NAME:-higress-apiserver}" initializeApiServer() { echo "Initializing API server configurations..." @@ -78,8 +79,8 @@ applySecretConfigmap() { apiVersion: v1 kind: Secret metadata: - name: higress-apiserver - namespace: higress-system + name: $SECRET_NAME + namespace: $NAMESPACE data: ca.crt: $(cat $VOLUMES_ROOT/api/ca.crt | base64 -w 0) ca.key: $(cat $VOLUMES_ROOT/api/ca.key | base64 -w 0) @@ -94,7 +95,7 @@ EOF apiVersion: v1 kind: ConfigMap metadata: - name: higress-config + name: higress-apiserver namespace: $NAMESPACE data: kubeconfig: | From b5fd8a7fc1e963a267ed7d6d7835a75f248424f1 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 00:26:19 +0800 Subject: [PATCH 13/34] rename make Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index bba2a65104..e6feb8f3d6 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -247,10 +247,10 @@ higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-buil # higress-conformance-test-nacos runs ingress nacos tests. .PHONY: higress-conformance-test-nacos -higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward run-higress-e2e-test-nacos delete-cluster +higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-nacos delete-cluster -.PHONY: port-forward -port-forward: +.PHONY: port-forward-nacos +port-forward-nacos: kubectl port-forward -n higress-system svc/nacos-service 8848:8848 & # create-cluster creates a kube cluster with kind. From 85b82743de694854f91a14c3332c39ba5dde1a84 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 00:30:34 +0800 Subject: [PATCH 14/34] opt import Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/utils/suite/suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 2b9ab048e5..d26ee11f37 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -15,11 +15,11 @@ package suite import ( "fmt" - "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" "testing" "github.com/alibaba/higress/test/e2e/conformance/utils/config" cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" + "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper" "github.com/stretchr/testify/require" From 370d38605b3b96dc4a2e7853a3d7cd23643799bd Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 12:49:30 +0800 Subject: [PATCH 15/34] update e2e Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 3 +- .../e2e/conformance/tests/configmap-global.go | 6 +-- test/e2e/conformance/tests/configmap-gzip.go | 11 +---- .../utils/configcenter/nacos/nacos.go | 8 +++- .../conformance/utils/configcenter/storage.go | 29 +++++++++++- test/e2e/conformance/utils/flags/flags.go | 2 +- .../e2e/conformance/utils/kubernetes/apply.go | 44 ++++++++++++++++--- .../conformance/utils/kubernetes/helpers.go | 29 ------------ test/e2e/conformance/utils/suite/suite.go | 20 ++++----- test/e2e/e2e_test.go | 2 +- tools/hack/conf/nacos.yaml | 21 ++++++--- tools/hack/gen-secret-configmap.sh | 2 +- 12 files changed, 105 insertions(+), 72 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index e6feb8f3d6..f13dbd4728 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -251,7 +251,8 @@ higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster dock .PHONY: port-forward-nacos port-forward-nacos: - kubectl port-forward -n higress-system svc/nacos-service 8848:8848 & + kubectl wait --timeout=10m -n higress-system deploy/nacos --for=condition=Available + kubectl port-forward -n higress-system svc/nacos-service 8848:8848 9848:9848 & # create-cluster creates a kube cluster with kind. .PHONY: create-cluster diff --git a/test/e2e/conformance/tests/configmap-global.go b/test/e2e/conformance/tests/configmap-global.go index c1eef1ecdf..ad26378607 100644 --- a/test/e2e/conformance/tests/configmap-global.go +++ b/test/e2e/conformance/tests/configmap-global.go @@ -19,7 +19,6 @@ import ( "github.com/alibaba/higress/pkg/ingress/kube/configmap" "github.com/alibaba/higress/test/e2e/conformance/utils/envoy" - "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/suite" ) @@ -548,10 +547,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ t.Run("ConfigMap Global Envoy", func(t *testing.T) { for _, testcase := range testCases { // apply config - err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig) - if err != nil { - t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") - } + suite.Applier.MustApplyConfigmapDataWithYaml(t, suite.ConfigCenter, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig, suite.EnableApiServer) t.Logf("Checking Envoy config for test case %s", testcase.name) for _, assertion := range testcase.envoyAssertion { envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, assertion) diff --git a/test/e2e/conformance/tests/configmap-gzip.go b/test/e2e/conformance/tests/configmap-gzip.go index 36caea2a60..2bf75ff2d6 100644 --- a/test/e2e/conformance/tests/configmap-gzip.go +++ b/test/e2e/conformance/tests/configmap-gzip.go @@ -20,7 +20,6 @@ import ( "github.com/alibaba/higress/pkg/ingress/kube/configmap" "github.com/alibaba/higress/test/e2e/conformance/utils/envoy" "github.com/alibaba/higress/test/e2e/conformance/utils/http" - "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/suite" ) @@ -296,10 +295,7 @@ var ConfigmapGzip = suite.ConformanceTest{ Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { t.Run("Configmap Gzip", func(t *testing.T) { for _, testcase := range testCases { - err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig) - if err != nil { - t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") - } + suite.Applier.MustApplyConfigmapDataWithYaml(t, suite.ConfigCenter, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig, suite.EnableApiServer) http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase.httpAssert) } }) @@ -315,10 +311,7 @@ var ConfigMapGzipEnvoy = suite.ConformanceTest{ t.Run("ConfigMap Gzip Envoy", func(t *testing.T) { for _, testcase := range testCases { // apply config - err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig) - if err != nil { - t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") - } + suite.Applier.MustApplyConfigmapDataWithYaml(t, suite.ConfigCenter, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig, suite.EnableApiServer) envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, testcase.envoyAssertion) } }) diff --git a/test/e2e/conformance/utils/configcenter/nacos/nacos.go b/test/e2e/conformance/utils/configcenter/nacos/nacos.go index 4ffa5e6fbc..a7249fec92 100644 --- a/test/e2e/conformance/utils/configcenter/nacos/nacos.go +++ b/test/e2e/conformance/utils/configcenter/nacos/nacos.go @@ -79,7 +79,9 @@ func NewClient(addr string) (cc.Storage, error) { }, nil } -func (s storage) PublishConfig(dataId, group, content string) error { +func (s storage) PublishConfig(kind, name, namespace, content string) error { + dataId := cc.GetDataId(kind, name) + group := namespace _, err := s.client.PublishConfig(vo.ConfigParam{ DataId: dataId, Group: group, @@ -88,7 +90,9 @@ func (s storage) PublishConfig(dataId, group, content string) error { return err } -func (s storage) DeleteConfig(dataId, group string) error { +func (s storage) DeleteConfig(kind, name, namespace string) error { + dataId := cc.GetDataId(kind, name) + group := namespace _, err := s.client.DeleteConfig(vo.ConfigParam{ DataId: dataId, Group: group, diff --git a/test/e2e/conformance/utils/configcenter/storage.go b/test/e2e/conformance/utils/configcenter/storage.go index d5c9e3bfab..dc64350eae 100644 --- a/test/e2e/conformance/utils/configcenter/storage.go +++ b/test/e2e/conformance/utils/configcenter/storage.go @@ -14,7 +14,32 @@ package cc +import "strings" + type Storage interface { - PublishConfig(dataId, group, content string) error - DeleteConfig(dataId, group string) error + PublishConfig(kind, name, namespace, content string) error + DeleteConfig(kind, name, namespace string) error +} + +func GetDataId(kind, name string) string { + switch strings.ToLower(kind) { + case "configmap": + kind = "configmaps" + case "secret": + kind = "secrets" + case "ingress": + kind = "ingresses" + case "service": + kind = "services" + case "ingressclass": + kind = "ingressclasses" + case "mcpbridge": + kind = "mcpbridges" + case "wasmplugin": + kind = "wasmplugins" + case "http2rpc": + kind = "http2rpcs" + default: + } + return kind + "." + name } diff --git a/test/e2e/conformance/utils/flags/flags.go b/test/e2e/conformance/utils/flags/flags.go index e70485a10a..cebae4c686 100644 --- a/test/e2e/conformance/utils/flags/flags.go +++ b/test/e2e/conformance/utils/flags/flags.go @@ -27,7 +27,7 @@ var ( WasmPluginType = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP") WasmPluginName = flag.String("wasmPluginName", "", "Define wasm plugin name") IsEnvoyConfigTest = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test") - EnableApiServer = flag.Bool("enableApiServer", false, "Determine if enable api server") + EnableApiServer = flag.Bool("enableApiServer", true, "Determine if enable api server") // TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace Storage = flag.String("storage", "http://127.0.0.1:8848", "Define storage") ) diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index 4861ce102f..14b9de8c67 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -19,9 +19,11 @@ import ( "errors" "fmt" "io" + v1 "k8s.io/api/core/v1" "net/http" "strings" "testing" + "time" ingress "github.com/alibaba/higress/test/e2e/conformance" cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" @@ -31,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" + yamlFormK8s "sigs.k8s.io/yaml" "github.com/alibaba/higress/test/e2e/conformance/utils/config" ) @@ -266,21 +269,52 @@ func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutCon for _, r := range resources { // TODO: Maybe add s or es - dataId := r.GetKind() + "es" - group := r.GetNamespace() - // convert r to string var content []byte content, err = r.MarshalJSON() require.NoError(t, err) // publish - err = cc.PublishConfig(dataId, group, string(content)) + err = cc.PublishConfig(r.GetKind(), r.GetName(), r.GetNamespace(), string(content)) require.NoError(t, err) if cleanup { t.Cleanup(func() { // delete - err = cc.DeleteConfig(dataId, group) + err = cc.DeleteConfig(r.GetKind(), r.GetName(), r.GetNamespace()) require.NoError(t, err) }) } } } + +// MustApplyConfigmapDataWithYaml apply configmap data with yaml +func (a Applier) MustApplyConfigmapDataWithYaml(t *testing.T, cc cc.Storage, c client.Client, namespace string, name string, key string, val any, enableApiServer bool) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cm := &v1.ConfigMap{} + err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, cm) + require.NoError(t, err) + + y, err := yamlFormK8s.Marshal(val) + require.NoError(t, err) + data := string(y) + + if cm.Data == nil { + cm.Data = make(map[string]string, 0) + } + cm.Data[key] = data + + t.Logf("🏗 Updating %s %s", name, namespace) + + if enableApiServer { + marshal, err := yamlFormK8s.Marshal(cm) + require.NoError(t, err) + err = cc.PublishConfig("configmap", cm.GetName(), cm.GetNamespace(), string(marshal)) + require.NoError(t, err) + return + } + + if err := c.Update(ctx, cm); err != nil { + require.NoError(t, err) + } + +} diff --git a/test/e2e/conformance/utils/kubernetes/helpers.go b/test/e2e/conformance/utils/kubernetes/helpers.go index ce002be242..fd91c9a617 100644 --- a/test/e2e/conformance/utils/kubernetes/helpers.go +++ b/test/e2e/conformance/utils/kubernetes/helpers.go @@ -15,7 +15,6 @@ package kubernetes import ( "context" - "sigs.k8s.io/yaml" "strings" "testing" "time" @@ -120,31 +119,3 @@ func FindPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName t.Logf("⌛️ %s was not in conditions list", condName) return false } - -func ApplyConfigmapDataWithYaml(t *testing.T, c client.Client, namespace string, name string, key string, val any) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cm := &v1.ConfigMap{} - if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, cm); err != nil { - return err - } - y, err := yaml.Marshal(val) - if err != nil { - return err - } - data := string(y) - - if cm.Data == nil { - cm.Data = make(map[string]string, 0) - } - cm.Data[key] = data - - t.Logf("🏗 Updating %s %s", name, namespace) - - if err := c.Update(ctx, cm); err != nil { - return err - } - - return nil -} diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index d26ee11f37..8435a26e4f 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -15,14 +15,14 @@ package suite import ( "fmt" + "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" + "istio.io/pkg/log" "testing" "github.com/alibaba/higress/test/e2e/conformance/utils/config" cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" - "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper" - "github.com/stretchr/testify/require" "istio.io/istio/pilot/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -137,20 +137,20 @@ func New(s Options) *ConformanceTestSuite { } } + if suite.EnableApiServer { + configClient, err := nacos.NewClient(suite.Storage) + if err != nil { + log.Fatalf("🚨 Failed to create config client: %v", err) + } + suite.ConfigCenter = configClient + } + return suite } // Setup ensures the base resources required for conformance tests are installed // in the cluster. It also ensures that all relevant resources are ready. func (suite *ConformanceTestSuite) Setup(t *testing.T) { - if suite.EnableApiServer { - t.Logf("📦 Test Setup: Ensuring ApiServer Storage has been accepted") - configClient, err := nacos.NewClient(suite.Storage) - require.NoError(t, err) - require.NotNil(t, configClient) - suite.ConfigCenter = configClient - } - t.Logf("📦 Test Setup: Ensuring IngressClass has been accepted") suite.Applier.IngressClass = suite.IngressClassName diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c1633b5fbd..6e1f2cf130 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -56,6 +56,6 @@ func TestHigressConformanceTests(t *testing.T) { IsEnvoyConfigTest: *flags.IsEnvoyConfigTest, }) - cSuite.Setup(t) + //cSuite.Setup(t) cSuite.Run(t, tests.ConformanceTests) } diff --git a/tools/hack/conf/nacos.yaml b/tools/hack/conf/nacos.yaml index 5c8fd61603..11093a60b8 100644 --- a/tools/hack/conf/nacos.yaml +++ b/tools/hack/conf/nacos.yaml @@ -25,18 +25,27 @@ kind: Namespace metadata: name: higress-system --- -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: nacos namespace: higress-system labels: name: nacos spec: - containers: - - name: nacos - image: swsk33/nacos-standalone:2.2.3 - ports: + replicas: 1 + selector: + matchLabels: + name: nacos + template: + metadata: + labels: + name: nacos + spec: + containers: + - name: nacos + image: swsk33/nacos-standalone:2.2.3 + ports: - containerPort: 8848 - containerPort: 9848 --- diff --git a/tools/hack/gen-secret-configmap.sh b/tools/hack/gen-secret-configmap.sh index ea95e7ac18..a3bd4f8e0a 100755 --- a/tools/hack/gen-secret-configmap.sh +++ b/tools/hack/gen-secret-configmap.sh @@ -90,7 +90,7 @@ data: nacos.key: $(cat $VOLUMES_ROOT/api/nacos.key | base64 -w 0) EOF - echo "Applying configmap $NAMESPACE/higress-config..." + echo "Applying configmap $NAMESPACE/higress-apiserver..." kubectl apply -f - < Date: Wed, 24 Jan 2024 12:54:54 +0800 Subject: [PATCH 16/34] opt import Signed-off-by: sjcsjc123 <1401189096@qq.com> --- helm/core/crds/customresourcedefinitions.gen.yaml | 5 +++++ test/e2e/conformance/utils/kubernetes/apply.go | 2 +- test/e2e/conformance/utils/suite/suite.go | 4 ++-- test/e2e/e2e_test.go | 2 +- tools/hack/gen-secret-configmap.sh | 3 +-- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/helm/core/crds/customresourcedefinitions.gen.yaml b/helm/core/crds/customresourcedefinitions.gen.yaml index 023ed35f9a..8b9c5f2c66 100644 --- a/helm/core/crds/customresourcedefinitions.gen.yaml +++ b/helm/core/crds/customresourcedefinitions.gen.yaml @@ -154,6 +154,11 @@ spec: type: array httpPath: type: string + paramFromEntireBody: + properties: + paramType: + type: string + type: object params: items: properties: diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index 14b9de8c67..4b5e899687 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "io" - v1 "k8s.io/api/core/v1" "net/http" "strings" "testing" @@ -28,6 +27,7 @@ import ( ingress "github.com/alibaba/higress/test/e2e/conformance" cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 8435a26e4f..4bac1990d8 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -15,15 +15,15 @@ package suite import ( "fmt" - "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" - "istio.io/pkg/log" "testing" "github.com/alibaba/higress/test/e2e/conformance/utils/config" cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" + "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper" "istio.io/istio/pilot/pkg/util/sets" + "istio.io/pkg/log" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6e1f2cf130..c1633b5fbd 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -56,6 +56,6 @@ func TestHigressConformanceTests(t *testing.T) { IsEnvoyConfigTest: *flags.IsEnvoyConfigTest, }) - //cSuite.Setup(t) + cSuite.Setup(t) cSuite.Run(t, tests.ConformanceTests) } diff --git a/tools/hack/gen-secret-configmap.sh b/tools/hack/gen-secret-configmap.sh index a3bd4f8e0a..bf82c76db9 100755 --- a/tools/hack/gen-secret-configmap.sh +++ b/tools/hack/gen-secret-configmap.sh @@ -19,7 +19,6 @@ RSA_KEY_LENGTH=4096 NAMESPACE="${NAMESPACE:-higress-system}" CLUSTER_NAME="${CLUSTER_NAME:-higress}" APISERVER_ADDRESS="${APISERVER_ADDRESS:-https://127.0.0.1:8443}" -SECRET_NAME="${SECRET_NAME:-higress-apiserver}" initializeApiServer() { echo "Initializing API server configurations..." @@ -79,7 +78,7 @@ applySecretConfigmap() { apiVersion: v1 kind: Secret metadata: - name: $SECRET_NAME + name: higress-apiserver namespace: $NAMESPACE data: ca.crt: $(cat $VOLUMES_ROOT/api/ca.crt | base64 -w 0) From 06651de62ab4fe207fa350d49b7c2bb794c82c4d Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 12:59:39 +0800 Subject: [PATCH 17/34] recover something Signed-off-by: sjcsjc123 <1401189096@qq.com> --- helm/core/crds/customresourcedefinitions.gen.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helm/core/crds/customresourcedefinitions.gen.yaml b/helm/core/crds/customresourcedefinitions.gen.yaml index 8b9c5f2c66..023ed35f9a 100644 --- a/helm/core/crds/customresourcedefinitions.gen.yaml +++ b/helm/core/crds/customresourcedefinitions.gen.yaml @@ -154,11 +154,6 @@ spec: type: array httpPath: type: string - paramFromEntireBody: - properties: - paramType: - type: string - type: object params: items: properties: From 35c5db1afa2d92d09f3d37e342d6661a7c76b828 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 13:11:56 +0800 Subject: [PATCH 18/34] modify default falg value Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/utils/flags/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/conformance/utils/flags/flags.go b/test/e2e/conformance/utils/flags/flags.go index cebae4c686..e70485a10a 100644 --- a/test/e2e/conformance/utils/flags/flags.go +++ b/test/e2e/conformance/utils/flags/flags.go @@ -27,7 +27,7 @@ var ( WasmPluginType = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP") WasmPluginName = flag.String("wasmPluginName", "", "Define wasm plugin name") IsEnvoyConfigTest = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test") - EnableApiServer = flag.Bool("enableApiServer", true, "Determine if enable api server") + EnableApiServer = flag.Bool("enableApiServer", false, "Determine if enable api server") // TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace Storage = flag.String("storage", "http://127.0.0.1:8848", "Define storage") ) From 944986220849161f6eb6c2a2626fcc80cce17a4d Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 13:15:27 +0800 Subject: [PATCH 19/34] make ci print simple Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index f13dbd4728..8b216dde80 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -310,4 +310,4 @@ run-higress-e2e-test-nacos: kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available - go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --enableApiServer=true \ No newline at end of file + go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true \ No newline at end of file From ecb2672b8834f0003a8f98a68b956d8e9695e168 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 15:10:59 +0800 Subject: [PATCH 20/34] update apply Signed-off-by: sjcsjc123 <1401189096@qq.com> --- docker/run-higress.sh | 4 +--- .../e2e/conformance/utils/kubernetes/apply.go | 19 +++++++++++++++++++ test/e2e/conformance/utils/suite/suite.go | 17 +++++++++++------ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/docker/run-higress.sh b/docker/run-higress.sh index 242bc87cd7..7444c4d961 100644 --- a/docker/run-higress.sh +++ b/docker/run-higress.sh @@ -24,6 +24,4 @@ done echo "starting higress..." -/usr/local/bin/higress "$@" - -echo "higress started." \ No newline at end of file +exec /usr/local/bin/higress "$@" \ No newline at end of file diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index 4b5e899687..57fbfdf72d 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -285,6 +285,25 @@ func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutCon } } +// MustDeleteConfig delete config from config center +func (a Applier) MustDeleteConfig(t *testing.T, timeoutConfig config.TimeoutConfig, location string, cc cc.Storage) { + data, err := getContentsFromPathOrURL(location, timeoutConfig) + require.NoError(t, err) + + decoder := yaml.NewYAMLOrJSONDecoder(data, 4096) + + resources, err := a.prepareResources(t, decoder) + if err != nil { + t.Logf("🧳 Manifest: %s", data.String()) + require.NoErrorf(t, err, "error parsing manifest") + } + + for _, r := range resources { + err = cc.DeleteConfig(r.GetKind(), r.GetName(), r.GetNamespace()) + require.NoError(t, err) + } +} + // MustApplyConfigmapDataWithYaml apply configmap data with yaml func (a Applier) MustApplyConfigmapDataWithYaml(t *testing.T, cc cc.Storage, c client.Client, namespace string, name string, key string, val any, enableApiServer bool) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 4bac1990d8..3ea09538a4 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -234,17 +234,22 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { t.Logf("🔥 Running Conformance Test: %s", test.ShortName) - for _, manifestLocation := range test.PreDeleteRs { - t.Logf("🧳 Applying PreDeleteRs Manifests: %s", manifestLocation) - suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, manifestLocation) - } - if suite.EnableApiServer { + for _, manifestLocation := range test.PreDeleteRs { + t.Logf("🧳 Applying PreDeleteRs ApiServer Storage Manifests: %s", manifestLocation) + suite.Applier.MustDeleteConfig(t, suite.TimeoutConfig, manifestLocation, suite.ConfigCenter) + } + for _, manifestLocation := range test.Manifests { - t.Logf("🧳 Applying ApiServer Storage Manifests") + t.Logf("🧳 Applying ApiServer Storage Manifests: %s", manifestLocation) suite.Applier.MustPublishConfig(t, suite.TimeoutConfig, manifestLocation, !test.NotCleanup, suite.ConfigCenter) } } else { + for _, manifestLocation := range test.PreDeleteRs { + t.Logf("🧳 Applying PreDeleteRs Manifests: %s", manifestLocation) + suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, manifestLocation) + } + for _, manifestLocation := range test.Manifests { t.Logf("🧳 Applying Manifests: %s", manifestLocation) suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup) From 62c421a57a17ec2c46360315332eebcda740aba1 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 16:16:59 +0800 Subject: [PATCH 21/34] update apply Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/utils/kubernetes/apply.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index 57fbfdf72d..9b78e71f9a 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -267,8 +267,8 @@ func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutCon require.NoErrorf(t, err, "error parsing manifest") } - for _, r := range resources { - // TODO: Maybe add s or es + for i := range resources { + r := resources[i] var content []byte content, err = r.MarshalJSON() require.NoError(t, err) @@ -278,6 +278,7 @@ func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutCon if cleanup { t.Cleanup(func() { // delete + t.Logf("🚮 Deleting %s %s", r.GetName(), r.GetKind()) err = cc.DeleteConfig(r.GetKind(), r.GetName(), r.GetNamespace()) require.NoError(t, err) }) @@ -298,7 +299,9 @@ func (a Applier) MustDeleteConfig(t *testing.T, timeoutConfig config.TimeoutConf require.NoErrorf(t, err, "error parsing manifest") } - for _, r := range resources { + for i := range resources { + r := resources[i] + t.Logf("🚮 Deleting %s %s", r.GetName(), r.GetKind()) err = cc.DeleteConfig(r.GetKind(), r.GetName(), r.GetNamespace()) require.NoError(t, err) } From 2e722592a022291edef7780bf5ef6524f9634d63 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 17:16:53 +0800 Subject: [PATCH 22/34] add wasm+nacos ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- .github/workflows/build-and-test-plugin.yaml | 36 ++++++++++++++++++++ Makefile.core.mk | 15 +++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-plugin.yaml b/.github/workflows/build-and-test-plugin.yaml index 53a0ddcb09..de44a45e34 100644 --- a/.github/workflows/build-and-test-plugin.yaml +++ b/.github/workflows/build-and-test-plugin.yaml @@ -61,6 +61,42 @@ jobs: - name: "Run Ingress WasmPlugins Tests" run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} make higress-wasmplugin-test + higress-wasmplugin-test-nacos: + runs-on: ubuntu-latest + strategy: + matrix: + wasmPluginType: [ GO ] + steps: + - uses: actions/checkout@v3 + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Setup Golang Caches + uses: actions/cache@v3 + with: + path: |- + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-go + + - name: Setup Submodule Caches + uses: actions/cache@v3 + with: + path: |- + .git/modules + key: ${{ runner.os }}-submodules-cache-${{ github.run_id }} + restore-keys: ${{ runner.os }}-submodules-cache + + - run: git stash # restore patch + + - name: "Run Ingress WasmPlugins Tests" + run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} make higress-wasmplugin-test-nacos + publish: runs-on: ubuntu-latest needs: [higress-wasmplugin-test] diff --git a/Makefile.core.mk b/Makefile.core.mk index 8b216dde80..ac567b7865 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -249,6 +249,10 @@ higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-buil .PHONY: higress-conformance-test-nacos higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-nacos delete-cluster +# higress-wasmplugin-test-nacos runs ingress wasmplugin nacos tests. +.PHONY: higress-wasmplugin-test-nacos +higress-wasmplugin-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-wasmplugin-nacos delete-cluster + .PHONY: port-forward-nacos port-forward-nacos: kubectl wait --timeout=10m -n higress-system deploy/nacos --for=condition=Available @@ -310,4 +314,13 @@ run-higress-e2e-test-nacos: kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available - go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true \ No newline at end of file + go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true + +.PHONY: run-higress-e2e-test-wasmplugin-nacos +run-higress-e2e-test-wasmplugin-nacos: + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available + @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available + go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=GO -wasmPluginName=request-block --ingress-class=higress --enableApiServer=true \ No newline at end of file From 0c83d5f5485f24c44f58e319c16ce488ca7485a1 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 17:21:52 +0800 Subject: [PATCH 23/34] update wasm+nacos ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- .github/workflows/build-and-test-plugin.yaml | 3 ++- Makefile.core.mk | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test-plugin.yaml b/.github/workflows/build-and-test-plugin.yaml index de44a45e34..9fe1b2736e 100644 --- a/.github/workflows/build-and-test-plugin.yaml +++ b/.github/workflows/build-and-test-plugin.yaml @@ -66,6 +66,7 @@ jobs: strategy: matrix: wasmPluginType: [ GO ] + wasmPluginName: [ request-block ] steps: - uses: actions/checkout@v3 @@ -95,7 +96,7 @@ jobs: - run: git stash # restore patch - name: "Run Ingress WasmPlugins Tests" - run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} make higress-wasmplugin-test-nacos + run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} PLUGIN_NAME=${{ matrix.wasmPluginName }} make higress-wasmplugin-test publish: runs-on: ubuntu-latest diff --git a/Makefile.core.mk b/Makefile.core.mk index ac567b7865..3312cab668 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -188,6 +188,10 @@ install-dev-nacos: pre-install kubectl apply -f tools/hack/conf/nacos.yaml tools/hack/gen-secret-configmap.sh helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' +install-dev-wasmplugin-nacos: build-wasmplugins pre-install + kubectl apply -f tools/hack/conf/nacos.yaml + tools/hack/gen-secret-configmap.sh + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' uninstall: helm uninstall higress -n higress-system @@ -251,7 +255,7 @@ higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster dock # higress-wasmplugin-test-nacos runs ingress wasmplugin nacos tests. .PHONY: higress-wasmplugin-test-nacos -higress-wasmplugin-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-wasmplugin-nacos delete-cluster +higress-wasmplugin-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin-nacos port-forward-nacos run-higress-e2e-test-wasmplugin-nacos delete-cluster .PHONY: port-forward-nacos port-forward-nacos: @@ -323,4 +327,4 @@ run-higress-e2e-test-wasmplugin-nacos: kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available - go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=GO -wasmPluginName=request-block --ingress-class=higress --enableApiServer=true \ No newline at end of file + go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --enableApiServer=true \ No newline at end of file From 0715626b462b363413019a7611a70408fe2ba052 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 17:48:34 +0800 Subject: [PATCH 24/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/base/service.yaml | 104 ++++++++++++++++++++++ test/e2e/conformance/utils/suite/suite.go | 5 ++ 2 files changed, 109 insertions(+) create mode 100644 test/e2e/conformance/base/service.yaml diff --git a/test/e2e/conformance/base/service.yaml b/test/e2e/conformance/base/service.yaml new file mode 100644 index 0000000000..67a964a277 --- /dev/null +++ b/test/e2e/conformance/base/service.yaml @@ -0,0 +1,104 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v1 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v2 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v2 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v3 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v3 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-echo-body-v1 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-echo-body-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: app-backend-v1 + namespace: higress-conformance-app-backend +spec: + selector: + app: app-backend-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: app-backend-v2 + namespace: higress-conformance-app-backend +spec: + selector: + app: app-backend-v2 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: web-backend + namespace: higress-conformance-web-backend +spec: + selector: + app: web-backend + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 3ea09538a4..5c88b07670 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -161,6 +161,11 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) { suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, baseManifest, suite.Cleanup) } + if suite.EnableApiServer { + t.Logf("📦 Test Setup: Applying ApiServer Storage Manifests") + suite.Applier.MustPublishConfig(t, suite.TimeoutConfig, "base/service.yaml", suite.Cleanup, suite.ConfigCenter) + } + t.Logf("📦 Test Setup: Applying programmatic resources") secret := kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-web-backend", "certificate", []string{"*"}) suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) From 62cb1dd672a6d646d11fc12ba920e1fc06ced9fa Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Wed, 24 Jan 2024 18:09:51 +0800 Subject: [PATCH 25/34] fix bug with wasm plugin Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/tests/cpp-wasm-basic-auth.go | 1 + test/e2e/conformance/tests/cpp-wasm-key-auth.go | 1 + test/e2e/conformance/tests/cpp-wasm-request-block.go | 1 + test/e2e/conformance/tests/go-wasm-basic-auth.go | 1 + test/e2e/conformance/tests/go-wasm-key-auth.go | 1 + test/e2e/conformance/tests/go-wasm-request-block.go | 1 + test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go | 1 + test/e2e/conformance/tests/go-wasm-sni-misdirect.go | 1 + test/e2e/conformance/tests/go-wasm-transformer.go | 1 + test/e2e/conformance/utils/flags/flags.go | 2 +- test/e2e/conformance/utils/suite/suite.go | 10 ++++++++++ 11 files changed, 20 insertions(+), 1 deletion(-) diff --git a/test/e2e/conformance/tests/cpp-wasm-basic-auth.go b/test/e2e/conformance/tests/cpp-wasm-basic-auth.go index 75d9753f6e..8ec8493f69 100644 --- a/test/e2e/conformance/tests/cpp-wasm-basic-auth.go +++ b/test/e2e/conformance/tests/cpp-wasm-basic-auth.go @@ -29,6 +29,7 @@ var CPPWasmPluginsBasicAuth = suite.ConformanceTest{ ShortName: "CPPWasmPluginsBasicAuth", Description: "The Ingress in the higress-conformance-infra namespace test the CPP basic-auth WASM plugin.", Manifests: []string{"tests/cpp-wasm-basic-auth.yaml"}, + PluginName: "basic-auth", Features: []suite.SupportedFeature{suite.WASMCPPConformanceFeature}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ diff --git a/test/e2e/conformance/tests/cpp-wasm-key-auth.go b/test/e2e/conformance/tests/cpp-wasm-key-auth.go index 1aeaa36dc0..d0ad8e83d0 100644 --- a/test/e2e/conformance/tests/cpp-wasm-key-auth.go +++ b/test/e2e/conformance/tests/cpp-wasm-key-auth.go @@ -29,6 +29,7 @@ var CPPWasmPluginsKeyAuth = suite.ConformanceTest{ ShortName: "CPPWasmPluginsKeyAuth", Description: "The Ingress in the higress-conformance-infra namespace test the CPP key_auth wasmplugins.", Manifests: []string{"tests/cpp-wasm-key-auth.yaml"}, + PluginName: "key-auth", Features: []suite.SupportedFeature{suite.WASMCPPConformanceFeature}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ diff --git a/test/e2e/conformance/tests/cpp-wasm-request-block.go b/test/e2e/conformance/tests/cpp-wasm-request-block.go index e00eba30f9..052cf18748 100644 --- a/test/e2e/conformance/tests/cpp-wasm-request-block.go +++ b/test/e2e/conformance/tests/cpp-wasm-request-block.go @@ -30,6 +30,7 @@ var CPPWasmPluginsRequestBlock = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the cpp request-block wasmplugins.", Manifests: []string{"tests/cpp-wasm-request-block.yaml"}, Features: []suite.SupportedFeature{suite.WASMCPPConformanceFeature}, + PluginName: "request-block", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-basic-auth.go b/test/e2e/conformance/tests/go-wasm-basic-auth.go index 3a67f61761..774615fc57 100644 --- a/test/e2e/conformance/tests/go-wasm-basic-auth.go +++ b/test/e2e/conformance/tests/go-wasm-basic-auth.go @@ -30,6 +30,7 @@ var WasmPluginsBasicAuth = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the basic-auth WASM plugin.", Manifests: []string{"tests/go-wasm-basic-auth.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "basic-auth", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-key-auth.go b/test/e2e/conformance/tests/go-wasm-key-auth.go index 3adb7ecab6..d7e48462b9 100644 --- a/test/e2e/conformance/tests/go-wasm-key-auth.go +++ b/test/e2e/conformance/tests/go-wasm-key-auth.go @@ -30,6 +30,7 @@ var WasmPluginsKeyAuth = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the key-auth WASM plugin.", Manifests: []string{"tests/go-wasm-key-auth.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "key-auth", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-request-block.go b/test/e2e/conformance/tests/go-wasm-request-block.go index 524205dbe0..a7e87255a0 100644 --- a/test/e2e/conformance/tests/go-wasm-request-block.go +++ b/test/e2e/conformance/tests/go-wasm-request-block.go @@ -30,6 +30,7 @@ var WasmPluginsRequestBlock = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the request-block wasmplugins.", Manifests: []string{"tests/go-wasm-request-block.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "request-block", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go b/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go index bf8d91434f..457ad234a4 100644 --- a/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go +++ b/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go @@ -30,6 +30,7 @@ var WasmPluginsJwtAuth = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the simple-jwt-auth wasmplugins.", Manifests: []string{"tests/go-wasm-simple-jwt-auth.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "simple-jwt-auth", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-sni-misdirect.go b/test/e2e/conformance/tests/go-wasm-sni-misdirect.go index f80204b6fc..0221dfa9c5 100644 --- a/test/e2e/conformance/tests/go-wasm-sni-misdirect.go +++ b/test/e2e/conformance/tests/go-wasm-sni-misdirect.go @@ -33,6 +33,7 @@ var WasmPluginsSniMisdirect = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the sni-misdirect wasmplugins.", Manifests: []string{"tests/go-wasm-sni-misdirect.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "sni-misdirect", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { // Prepare certificates and secrets for testcases caCertOut, _, caCert, caKey := cert.MustGenerateCaCert(t) diff --git a/test/e2e/conformance/tests/go-wasm-transformer.go b/test/e2e/conformance/tests/go-wasm-transformer.go index b323cdea0a..47099545cb 100644 --- a/test/e2e/conformance/tests/go-wasm-transformer.go +++ b/test/e2e/conformance/tests/go-wasm-transformer.go @@ -31,6 +31,7 @@ var WasmPluginsTransformer = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the transformer WASM plugin.", Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, Manifests: []string{"tests/go-wasm-transformer.yaml"}, + PluginName: "transformer", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/utils/flags/flags.go b/test/e2e/conformance/utils/flags/flags.go index e70485a10a..04c04a2284 100644 --- a/test/e2e/conformance/utils/flags/flags.go +++ b/test/e2e/conformance/utils/flags/flags.go @@ -25,7 +25,7 @@ var ( ExemptFeatures = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites") IsWasmPluginTest = flag.Bool("isWasmPluginTest", false, "Determine if run wasm plugin conformance test") WasmPluginType = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP") - WasmPluginName = flag.String("wasmPluginName", "", "Define wasm plugin name") + WasmPluginName = flag.String("wasmPluginName", "all", "Define wasm plugin name") IsEnvoyConfigTest = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test") EnableApiServer = flag.Bool("enableApiServer", false, "Determine if enable api server") // TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 5c88b07670..76e21fbb83 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -44,6 +44,7 @@ type ConformanceTestSuite struct { EnableApiServer bool Storage string ConfigCenter cc.Storage + WASMOptions } // Options can be used to initialize a ConformanceTestSuite. @@ -124,6 +125,7 @@ func New(s Options) *ConformanceTestSuite { TimeoutConfig: s.TimeoutConfig, EnableApiServer: s.EnableApiServer, Storage: s.Storage, + WASMOptions: s.WASMOptions, } // apply defaults @@ -210,6 +212,7 @@ type ConformanceTest struct { Description string PreDeleteRs []string Manifests []string + PluginName string Features []SupportedFeature Slow bool Parallel bool @@ -237,6 +240,13 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { t.Skipf("🏊🏼 Skipping %s: test explicitly skipped", test.ShortName) } + // Skip wasm plugin name test + if suite.IsWasmPluginTest { + if suite.WasmPluginName != "all" && suite.WasmPluginName != test.PluginName { + t.Skipf("🏊🏼 Skipping %s: wasm plugin name not match", test.ShortName) + } + } + t.Logf("🔥 Running Conformance Test: %s", test.ShortName) if suite.EnableApiServer { From 4b5cd88fa67857736c49249a265aaae05cb070b3 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 1 Feb 2024 13:57:24 +0800 Subject: [PATCH 26/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index d2edd4fccf..34349ff37e 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -371,7 +371,7 @@ run-higress-e2e-test-wasmplugin: # run-higress-e2e-test-wasmplugin-run starts to run ingress e2e conformance tests. .PHONY: run-higress-e2e-test-wasmplugin-run run-higress-e2e-test-wasmplugin-run: - @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" From 93497720a1d2727eccd1e717d898041d6a38da34 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 1 Feb 2024 23:47:39 +0800 Subject: [PATCH 27/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 2b357f0cdd..f337e1fbc0 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -381,28 +381,28 @@ run-higress-e2e-test-wasmplugin-run: # run-higress-e2e-test-wasmplugin-clean starts to clean ingress e2e tests. .PHONY: run-higress-e2e-test-wasmplugin-clean run-higress-e2e-test-wasmplugin-clean: - @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available - go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=clean + go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=clean .PHONY: run-higress-e2e-test-nacos run-higress-e2e-test-nacos: - @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available - go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true + go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true .PHONY: run-higress-e2e-test-wasmplugin-nacos run-higress-e2e-test-wasmplugin-nacos: - @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available - go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --enableApiServer=true + go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --enableApiServer=true From a3b114b86619d1fe4c561b391227a9f63427353f Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 2 Feb 2024 01:11:35 +0800 Subject: [PATCH 28/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- test/e2e/conformance/utils/kubernetes/helpers.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/e2e/conformance/utils/kubernetes/helpers.go b/test/e2e/conformance/utils/kubernetes/helpers.go index 7481e38f34..fd91c9a617 100644 --- a/test/e2e/conformance/utils/kubernetes/helpers.go +++ b/test/e2e/conformance/utils/kubernetes/helpers.go @@ -19,8 +19,6 @@ import ( "testing" "time" - "sigs.k8s.io/yaml" - "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -121,4 +119,3 @@ func FindPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName t.Logf("⌛️ %s was not in conditions list", condName) return false } - From 6cd64b449b49abc66e86e4ac12aff3063e7dfe26 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 29 Feb 2024 15:22:40 +0800 Subject: [PATCH 29/34] update nacos image Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 26 ++--- helm/core/templates/nacos-service.yaml | 30 +++++ helm/core/templates/nacos-statefulset.yaml | 104 ++++++++++++++++++ helm/core/values.yaml | 34 +++++- test/e2e/conformance/base/nacos.yaml | 2 +- .../e2e/conformance/utils/kubernetes/apply.go | 6 +- tools/hack/conf/nacos.yaml | 65 ----------- 7 files changed, 183 insertions(+), 84 deletions(-) create mode 100644 helm/core/templates/nacos-service.yaml create mode 100644 helm/core/templates/nacos-statefulset.yaml delete mode 100644 tools/hack/conf/nacos.yaml diff --git a/Makefile.core.mk b/Makefile.core.mk index 6ddbced47a..9419797525 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -185,13 +185,11 @@ install-dev: pre-install install-dev-wasmplugin: build-wasmplugins pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' install-dev-nacos: pre-install - kubectl apply -f tools/hack/conf/nacos.yaml tools/hack/gen-secret-configmap.sh - helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'nacos.enabled=true' install-dev-wasmplugin-nacos: build-wasmplugins pre-install - kubectl apply -f tools/hack/conf/nacos.yaml tools/hack/gen-secret-configmap.sh - helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'nacos.enabled=true' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' uninstall: helm uninstall higress -n higress-system @@ -275,8 +273,8 @@ higress-wasmplugin-test-nacos: $(tools/kind) delete-cluster create-cluster docke .PHONY: port-forward-nacos port-forward-nacos: - kubectl wait --timeout=10m -n higress-system deploy/nacos --for=condition=Available - kubectl port-forward -n higress-system svc/nacos-service 8848:8848 9848:9848 & + kubectl wait --timeout=10m -n higress-system --for=condition=Ready pod -l app.kubernetes.io/name=higress-nacos,app.kubernetes.io/instance=higress + kubectl port-forward -n higress-system svc/higress-nacos-service 8848:8848 9848:9848 & # create-cluster creates a kube cluster with kind. .PHONY: create-cluster @@ -296,13 +294,13 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/docker-pull-image.sh docker.io/swsk33/nacos-standalone 2.2.3 + tools/hack/docker-pull-image.sh docker.io/nacos/nacos-server latest tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.3 + tools/hack/kind-load-image.sh docker.io/nacos/nacos-server latest tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 @@ -367,8 +365,8 @@ run-higress-e2e-test-wasmplugin: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=all - - # run-higress-e2e-test-wasmplugin-run starts to run ingress e2e conformance tests. + +# run-higress-e2e-test-wasmplugin-run starts to run ingress e2e conformance tests. .PHONY: run-higress-e2e-test-wasmplugin-run run-higress-e2e-test-wasmplugin-run: @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @@ -377,8 +375,8 @@ run-higress-e2e-test-wasmplugin-run: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=run - - # run-higress-e2e-test-wasmplugin-clean starts to clean ingress e2e tests. + +# run-higress-e2e-test-wasmplugin-clean starts to clean ingress e2e tests. .PHONY: run-higress-e2e-test-wasmplugin-clean run-higress-e2e-test-wasmplugin-clean: @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @@ -387,7 +385,7 @@ run-higress-e2e-test-wasmplugin-clean: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=clean - + .PHONY: run-higress-e2e-test-nacos run-higress-e2e-test-nacos: @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @@ -396,7 +394,7 @@ run-higress-e2e-test-nacos: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true - + .PHONY: run-higress-e2e-test-wasmplugin-nacos run-higress-e2e-test-wasmplugin-nacos: @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" diff --git a/helm/core/templates/nacos-service.yaml b/helm/core/templates/nacos-service.yaml new file mode 100644 index 0000000000..d2c1a43b8b --- /dev/null +++ b/helm/core/templates/nacos-service.yaml @@ -0,0 +1,30 @@ +{{- if .Values.nacos.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.nacos.name }}-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.nacos.service.type }} + ports: + - port: {{ .Values.nacos.service.port }} + targetPort: {{ .Values.nacos.service.port }} + protocol: TCP + name: http + - port: {{ add .Values.nacos.service.port 1000}} + name: client-rpc + targetPort: {{add .Values.nacos.service.port 1000}} + - port: {{add .Values.nacos.service.port 1001}} + name: raft-rpc + targetPort: {{add .Values.nacos.service.port 1001}} + - port: 7848 + name: old-raft-rpc + targetPort: 7848 + protocol: TCP + {{- if eq .Values.nacos.service.type "NodePort" }} + nodePort: {{ .Values.nacos.service.nodePort }} + {{- end }} + selector: + app.kubernetes.io/name: {{ .Values.nacos.name }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end}} \ No newline at end of file diff --git a/helm/core/templates/nacos-statefulset.yaml b/helm/core/templates/nacos-statefulset.yaml new file mode 100644 index 0000000000..3d76e0de50 --- /dev/null +++ b/helm/core/templates/nacos-statefulset.yaml @@ -0,0 +1,104 @@ +{{- if .Values.nacos.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Values.nacos.name }} + namespace: {{ .Release.Namespace }} + annotations: + {{- toYaml .Values.nacos.annotations | indent 4 }} +spec: + serviceName: {{ .Values.nacos.name }}-service + replicas: {{ .Values.nacos.replica }} + {{- if .Values.nacos.podManagementPolicy }} + podManagementPolicy: {{ .Values.nacos.podManagementPolicy }} + {{- else}} + podManagementPolicy: OrderedReady + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ .Values.nacos.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Values.nacos.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + {{- with .Values.nacos.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nacos.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nacos.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Values.nacos.name }} + image: "{{ .Values.nacos.hub }}:{{ .Values.nacos.tag }}" + imagePullPolicy: {{ .Values.nacos.pullPolicy }} + startupProbe: + initialDelaySeconds: 180 + periodSeconds: 5 + timeoutSeconds: 10 + httpGet: + scheme: HTTP + port: {{ .Values.nacos.service.port }} + path: /nacos/v1/console/health/readiness + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 10 + httpGet: + scheme: HTTP + port: {{ .Values.nacos.service.port }} + path: /nacos/v1/console/health/liveness + ports: + - name: http + containerPort: {{ .Values.nacos.service.port }} + protocol: TCP + - containerPort: {{ add .Values.nacos.service.port 1000}} + name: client-rpc + - containerPort: {{ add .Values.nacos.service.port 1001 }} + name: raft-rpc + - containerPort: 7848 + name: old-raft-rpc + resources: + {{- toYaml .Values.nacos.resources | nindent 12 }} + env: + - name: NACOS_SERVER_PORT + value: {{ .Values.nacos.service.port | quote }} + - name: NACOS_APPLICATION_PORT + value: {{ .Values.nacos.service.port | quote }} + - name: PREFER_HOST_MODE + value: {{ .Values.nacos.preferHostMode | quote }} + - name: MODE + value: "standalone" + - name: EMBEDDED_STORAGE + value: embedded + volumeMounts: + - name: data + mountPath: /home/nacos/plugins/peer-finder + subPath: peer-finder + - name: data + mountPath: /home/nacos/data + subPath: data + - name: data + mountPath: /home/nacos/logs + subPath: logs + {{- if not .Values.nacos.persistence.enabled }} + volumes: + - name: data + emptyDir: { } + {{- end }} + {{- if .Values.nacos.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + spec: + {{- toYaml .Values.nacos.persistence.data | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/core/values.yaml b/helm/core/values.yaml index bf2be55018..fa285339f0 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -651,7 +651,7 @@ apiserver: imagePullPolicy: IfNotPresent securePort: 8443 storage: nacos - serverAddr: http://127.0.0.1:8848 + serverAddr: http://higress-nacos-service.higress-system.svc.cluster.local:8848 username: "" password: "" namespaceID: "" @@ -663,3 +663,35 @@ skywalking: service: address: ~ port: 11800 + +# nacos config settings +nacos: + name: "higress-nacos" + enabled: false + hub: nacos/nacos-server + tag: latest + pullPolicy: IfNotPresent + replica: 1 + podManagementPolicy: Parallel + nodeSelector: { } + affinity: { } + tolerations: [ ] + persistence: + enabled: false + data: + accessModes: + - ReadWriteOnce + storageClassName: manual + resources: + requests: + storage: 5Gi + service: + port: 8848 + type: NodePort + nodePort: 30000 + annotations: {} + preferHostMode: hostname + resources: + requests: + cpu: 500m + memory: 2Gi diff --git a/test/e2e/conformance/base/nacos.yaml b/test/e2e/conformance/base/nacos.yaml index c2af14e536..777b325e52 100644 --- a/test/e2e/conformance/base/nacos.yaml +++ b/test/e2e/conformance/base/nacos.yaml @@ -31,7 +31,7 @@ metadata: spec: containers: - name: nacos-standlone-rc3 - image: swsk33/nacos-standalone:2.2.3 + image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 ports: - containerPort: 8848 - containerPort: 9848 diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index 005c92aa97..98b28bde26 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" - yamlFormK8s "sigs.k8s.io/yaml" + yamlFromK8s "sigs.k8s.io/yaml" "github.com/alibaba/higress/test/e2e/conformance/utils/config" ) @@ -316,7 +316,7 @@ func (a Applier) MustApplyConfigmapDataWithYaml(t *testing.T, cc cc.Storage, c c err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, cm) require.NoError(t, err) - y, err := yamlFormK8s.Marshal(val) + y, err := yamlFromK8s.Marshal(val) require.NoError(t, err) data := string(y) @@ -328,7 +328,7 @@ func (a Applier) MustApplyConfigmapDataWithYaml(t *testing.T, cc cc.Storage, c c t.Logf("🏗 Updating %s %s", name, namespace) if enableApiServer { - marshal, err := yamlFormK8s.Marshal(cm) + marshal, err := yamlFromK8s.Marshal(cm) require.NoError(t, err) err = cc.PublishConfig("configmap", cm.GetName(), cm.GetNamespace(), string(marshal)) require.NoError(t, err) diff --git a/tools/hack/conf/nacos.yaml b/tools/hack/conf/nacos.yaml deleted file mode 100644 index 11093a60b8..0000000000 --- a/tools/hack/conf/nacos.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2022 Alibaba Group Holding Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file contains the base resources that most conformance tests will rely -# on. This includes 3 namespaces along with Gateways, Services and Deployments -# that can be used as backends for routing traffic. The most important -# resources included are the Gateways (all in the higress-conformance-infra -# namespace): -# - same-namespace (only supports route in same ns) -# - all-namespaces (supports routes in all ns) -# - backend-namespaces (supports routes in ns with backend label) -apiVersion: v1 -kind: Namespace -metadata: - name: higress-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nacos - namespace: higress-system - labels: - name: nacos -spec: - replicas: 1 - selector: - matchLabels: - name: nacos - template: - metadata: - labels: - name: nacos - spec: - containers: - - name: nacos - image: swsk33/nacos-standalone:2.2.3 - ports: - - containerPort: 8848 - - containerPort: 9848 ---- -apiVersion: v1 -kind: Service -metadata: - name: nacos-service - namespace: higress-system -spec: - selector: - name: nacos - clusterIP: None - ports: - - name: foo # name is not required for single-port Services - port: 8848 - - name: bar - port: 9848 From 5575512480731812430e482e3237d893ab6946a5 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 29 Feb 2024 15:30:03 +0800 Subject: [PATCH 30/34] update nacos image Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 ++ test/e2e/conformance/base/nacos.yaml | 3 --- test/e2e/conformance/utils/flags/flags.go | 2 +- test/e2e/conformance/utils/suite/suite.go | 16 ++++++++-------- test/e2e/e2e_test.go | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 9419797525..5f2dd77153 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -294,12 +294,14 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/docker-pull-image.sh docker.io/nacos/nacos-server latest tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/kind-load-image.sh docker.io/nacos/nacos-server latest tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 diff --git a/test/e2e/conformance/base/nacos.yaml b/test/e2e/conformance/base/nacos.yaml index 777b325e52..c2e2c9e6d4 100644 --- a/test/e2e/conformance/base/nacos.yaml +++ b/test/e2e/conformance/base/nacos.yaml @@ -34,7 +34,6 @@ spec: image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 ports: - containerPort: 8848 - - containerPort: 9848 --- apiVersion: v1 kind: Service @@ -48,5 +47,3 @@ spec: ports: - name: foo # name is not required for single-port Services port: 8848 - - name: bar - port: 9848 diff --git a/test/e2e/conformance/utils/flags/flags.go b/test/e2e/conformance/utils/flags/flags.go index 733d6d4e7d..d13db59cb2 100644 --- a/test/e2e/conformance/utils/flags/flags.go +++ b/test/e2e/conformance/utils/flags/flags.go @@ -30,5 +30,5 @@ var ( TestArea = flag.String("test-area", "all", "Test area to run, like all to run setup/run/clean, setup to prepare test environment, run to run test cases, clean to clean test environment") EnableApiServer = flag.Bool("enableApiServer", false, "Determine if enable api server") // TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace - Storage = flag.String("storage", "http://127.0.0.1:8848", "Define storage") + ApiServerStorage = flag.String("storage", "http://127.0.0.1:8848", "Define storage") ) diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index 0fe915a2f2..fc1c428042 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -49,7 +49,7 @@ type ConformanceTestSuite struct { TimeoutConfig config.TimeoutConfig SupportedFeatures sets.Set EnableApiServer bool - Storage string + ApiServerStorage string ConfigCenter cc.Storage WASMOptions } @@ -78,8 +78,8 @@ type Options struct { // IsEnvoyConfigTest indicates whether or not the test is for envoy config IsEnvoyConfigTest bool - EnableApiServer bool - Storage string + EnableApiServer bool + ApiServerStorage string } type WASMOptions struct { @@ -129,10 +129,10 @@ func New(s Options) *ConformanceTestSuite { Applier: kubernetes.Applier{ NamespaceLabels: s.NamespaceLabels, }, - TimeoutConfig: s.TimeoutConfig, - EnableApiServer: s.EnableApiServer, - Storage: s.Storage, - WASMOptions: s.WASMOptions, + TimeoutConfig: s.TimeoutConfig, + EnableApiServer: s.EnableApiServer, + ApiServerStorage: s.ApiServerStorage, + WASMOptions: s.WASMOptions, } // apply defaults @@ -148,7 +148,7 @@ func New(s Options) *ConformanceTestSuite { } if suite.EnableApiServer { - configClient, err := nacos.NewClient(suite.Storage) + configClient, err := nacos.NewClient(suite.ApiServerStorage) if err != nil { log.Fatalf("🚨 Failed to create config client: %v", err) } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 71b617bf43..16e05c4dbd 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -53,7 +53,7 @@ func TestHigressConformanceTests(t *testing.T) { GatewayAddress: "localhost", EnableAllSupportedFeatures: true, EnableApiServer: *flags.EnableApiServer, - Storage: *flags.Storage, + ApiServerStorage: *flags.ApiServerStorage, IsEnvoyConfigTest: *flags.IsEnvoyConfigTest, }) From 81810c0169fe9b2a5c91b49cd0dc2ca1be13953b Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 29 Feb 2024 16:36:44 +0800 Subject: [PATCH 31/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 -- test/e2e/conformance/base/nacos.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 5f2dd77153..9419797525 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -294,14 +294,12 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/docker-pull-image.sh docker.io/nacos/nacos-server latest tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/kind-load-image.sh docker.io/nacos/nacos-server latest tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 diff --git a/test/e2e/conformance/base/nacos.yaml b/test/e2e/conformance/base/nacos.yaml index c2e2c9e6d4..ef38753a57 100644 --- a/test/e2e/conformance/base/nacos.yaml +++ b/test/e2e/conformance/base/nacos.yaml @@ -31,7 +31,7 @@ metadata: spec: containers: - name: nacos-standlone-rc3 - image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 + image: docker.io/nacos/nacos-server:latest ports: - containerPort: 8848 --- From daba6c71da3fbd5ca5b194b053d3442a2228665c Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 29 Feb 2024 17:47:53 +0800 Subject: [PATCH 32/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- .github/workflows/build-and-test-plugin.yaml | 10 ++++++++++ .github/workflows/build-and-test.yaml | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.github/workflows/build-and-test-plugin.yaml b/.github/workflows/build-and-test-plugin.yaml index cb03b67838..65bfef12ea 100644 --- a/.github/workflows/build-and-test-plugin.yaml +++ b/.github/workflows/build-and-test-plugin.yaml @@ -85,6 +85,16 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: "Setup Go" uses: actions/setup-go@v3 with: diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 8b603522c5..714be2b111 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -106,6 +106,16 @@ jobs: needs: [build] steps: - uses: actions/checkout@v3 + + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true - name: "Setup Go" uses: actions/setup-go@v3 @@ -141,6 +151,16 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: "Setup Go" uses: actions/setup-go@v3 with: From 32a67861baa60875127f17cd3bc99e801d58a9bc Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 1 Mar 2024 14:14:13 +0800 Subject: [PATCH 33/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 ++ test/e2e/conformance/base/nacos.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 9419797525..ada3b0f464 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -294,12 +294,14 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/docker-pull-image.sh docker.io/nacos/nacos-server latest tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 + tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/kind-load-image.sh docker.io/nacos/nacos-server latest tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 diff --git a/test/e2e/conformance/base/nacos.yaml b/test/e2e/conformance/base/nacos.yaml index ef38753a57..c2e2c9e6d4 100644 --- a/test/e2e/conformance/base/nacos.yaml +++ b/test/e2e/conformance/base/nacos.yaml @@ -31,7 +31,7 @@ metadata: spec: containers: - name: nacos-standlone-rc3 - image: docker.io/nacos/nacos-server:latest + image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 ports: - containerPort: 8848 --- From 0a425dd63f4cc01b2958bc34ceee93d159952f39 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 1 Mar 2024 14:14:44 +0800 Subject: [PATCH 34/34] fix ci Signed-off-by: sjcsjc123 <1401189096@qq.com> --- Makefile.core.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index ada3b0f464..5f2dd77153 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -301,7 +301,7 @@ kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster us tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 - tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 tools/hack/kind-load-image.sh docker.io/nacos/nacos-server latest tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2