Merge branch 'main' into 222

.github/workflows/lint.yml

@@ -1,8 +1,9 @@
 name: lint
 
 on:
-  push:
-    branches: [ "main" ]
+  #FIXME: uncomment it once open source
+  #push:
+    #branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
 

.github/workflows/test.yml

@@ -1,11 +1,12 @@
 name: test
 
 on:
-  push:
-    branches: [ "main" ]
-    paths-ignore:
-    - "site/**"
-    - "**/*.md"
+  #FIXME: uncomment it once open source
+  #push:
+    #branches: [ "main" ]
+    #paths-ignore:
+    #- "site/**"
+    #- "**/*.md"
   pull_request:
     branches: [ "main" ]
     paths-ignore:

controller/tests/benchmark/httpfilterpolicy_controller_test.go

@@ -79,21 +79,21 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 		var policies mosniov1.HTTPFilterPolicyList
 		if err := k8sClient.List(ctx, &policies); err == nil {
 			for _, e := range policies.Items {
-				Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, &e)
 			}
 		}
 
 		var virtualservices istiov1b1.VirtualServiceList
 		if err := k8sClient.List(ctx, &virtualservices); err == nil {
 			for _, e := range virtualservices.Items {
-				Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, e)
 			}
 		}
 
 		var gateways istiov1b1.GatewayList
 		if err := k8sClient.List(ctx, &gateways); err == nil {
 			for _, e := range gateways.Items {
-				Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, e)
 			}
 		}
 	})

controller/tests/integration/controller/consumer_controller_test.go

@@ -50,14 +50,14 @@ var _ = Describe("Consumer controller", func() {
 			var consumers mosniov1.ConsumerList
 			if err := k8sClient.List(ctx, &consumers); err == nil {
 				for _, e := range consumers.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var envoyfilters istiov1a3.EnvoyFilterList
 			if err := k8sClient.List(ctx, &envoyfilters); err == nil {
 				for _, e := range envoyfilters.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 		})

controller/tests/integration/controller/httpfilterpolicy_controller_test.go

@@ -76,7 +76,7 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 			var policies mosniov1.HTTPFilterPolicyList
 			if err := k8sClient.List(ctx, &policies); err == nil {
 				for _, e := range policies.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 		})
@@ -159,28 +159,28 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 			var policies mosniov1.HTTPFilterPolicyList
 			if err := k8sClient.List(ctx, &policies); err == nil {
 				for _, e := range policies.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var virtualservices istiov1b1.VirtualServiceList
 			if err := k8sClient.List(ctx, &virtualservices); err == nil {
 				for _, e := range virtualservices.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 
 			var gateways istiov1b1.GatewayList
 			if err := k8sClient.List(ctx, &gateways); err == nil {
 				for _, e := range gateways.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 
 			var envoyfilters istiov1a3.EnvoyFilterList
 			if err := k8sClient.List(ctx, &envoyfilters); err == nil {
 				for _, e := range envoyfilters.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 
@@ -585,28 +585,28 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 			var policies mosniov1.HTTPFilterPolicyList
 			if err := k8sClient.List(ctx, &policies); err == nil {
 				for _, e := range policies.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var httproutes gwapiv1.HTTPRouteList
 			if err := k8sClient.List(ctx, &httproutes); err == nil {
 				for _, e := range httproutes.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var gateways gwapiv1.GatewayList
 			if err := k8sClient.List(ctx, &gateways); err == nil {
 				for _, e := range gateways.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var envoyfilters istiov1a3.EnvoyFilterList
 			if err := k8sClient.List(ctx, &envoyfilters); err == nil {
 				for _, e := range envoyfilters.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 

controller/tests/integration/controller/serviceregistry_controller_test.go

@@ -46,7 +46,7 @@ var _ = Describe("ServiceRegistry controller", func() {
 			var registries mosniov1.ServiceRegistryList
 			if err := k8sClient.List(ctx, &registries); err == nil {
 				for _, e := range registries.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 

controller/tests/integration/registries/nacos_test.go

@@ -138,7 +138,7 @@ var _ = Describe("Nacos", func() {
 		var registries mosniov1.ServiceRegistryList
 		if err := k8sClient.List(ctx, &registries); err == nil {
 			for _, e := range registries.Items {
-				Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, &e)
 			}
 		}
 
@@ -242,7 +242,7 @@ var _ = Describe("Nacos", func() {
 		Expect(k8sClient.Patch(ctx, currNacos, base)).Should(Succeed())
 		Eventually(func() bool {
 			entries = listServiceEntries()
-			return len(entries) == 2
+			return len(entries) == 1
 		}, timeout, interval).Should(BeTrue())
 
 		// subscribe change

controller/tests/pkg/utils.go

@@ -15,13 +15,16 @@
 package pkg
 
 import (
+	"context"
 	"encoding/json"
 	"strings"
 	"testing"
 
+	. "github.com/onsi/gomega"
 	"github.com/stretchr/testify/require"
 	istiov1a3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
 	istiov1b1 "istio.io/client-go/pkg/apis/networking/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
@@ -74,3 +77,10 @@ func FakeK8sClient(t *testing.T) client.Client {
 	require.NoError(t, err)
 	return k8sClient
 }
+
+func DeleteK8sResource(ctx context.Context, k8sClient client.Client, obj client.Object, opts ...client.DeleteOption) {
+	err := k8sClient.Delete(ctx, obj, opts...)
+	if err != nil && !apierrors.IsNotFound(err) {
+		Expect(err).Should(Succeed())
+	}
+}

e2e/pkg/k8s/k8s.go

@@ -24,12 +24,15 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	istiov1b1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
+	mosniov1 "mosn.io/htnn/controller/api/v1"
 	"mosn.io/htnn/pkg/log"
 )
 
@@ -100,3 +103,43 @@ func readResources(decoder *yaml.YAMLOrJSONDecoder) ([]unstructured.Unstructured
 
 	return resources, nil
 }
+
+func deleteResource(t *testing.T, ctx context.Context, k8sClient client.Client, obj client.Object, opts ...client.DeleteOption) {
+	err := k8sClient.Delete(ctx, obj, opts...)
+	if err != nil && !apierrors.IsNotFound(err) {
+		require.NoError(t, err)
+	}
+	logger.Info("Deleted", "name", obj.GetName(), "kind", obj.GetObjectKind())
+}
+
+func CleanUp(t *testing.T, c client.Client) {
+	ctx := context.Background()
+	var policies mosniov1.HTTPFilterPolicyList
+	err := c.List(ctx, &policies)
+	require.NoError(t, err)
+	for _, e := range policies.Items {
+		deleteResource(t, ctx, c, &e)
+	}
+
+	var consumers mosniov1.ConsumerList
+	err = c.List(ctx, &consumers)
+	require.NoError(t, err)
+	for _, e := range consumers.Items {
+		deleteResource(t, ctx, c, &e)
+	}
+
+	var httproutes gwapiv1.HTTPRouteList
+	err = c.List(ctx, &httproutes)
+	require.NoError(t, err)
+	for _, e := range httproutes.Items {
+		deleteResource(t, ctx, c, &e)
+	}
+
+	var virtualservices istiov1b1.VirtualServiceList
+	err = c.List(ctx, &virtualservices)
+	require.NoError(t, err)
+	for _, e := range virtualservices.Items {
+		deleteResource(t, ctx, c, e)
+	}
+	// let HTNN to clean up EnvoyFilter
+}

e2e/pkg/suite/suite.go

@@ -31,13 +31,10 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-	istiov1b1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
 
-	mosniov1 "mosn.io/htnn/controller/api/v1"
 	"mosn.io/htnn/e2e/pkg/k8s"
 	"mosn.io/htnn/pkg/log"
 )
@@ -145,42 +142,7 @@ func (suite *Suite) stopPortForward(t *testing.T) {
 }
 
 func (suite *Suite) cleanup(t *testing.T) {
-	c := suite.Opt.Client
-	ctx := context.Background()
-	var policies mosniov1.HTTPFilterPolicyList
-	err := c.List(ctx, &policies)
-	require.NoError(t, err)
-	for _, e := range policies.Items {
-		// DeepCopy here is to satisfy the requirement of gosec
-		require.NoError(t, c.Delete(ctx, e.DeepCopy()))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-
-	var consumers mosniov1.ConsumerList
-	err = c.List(ctx, &consumers)
-	require.NoError(t, err)
-	for _, e := range consumers.Items {
-		// DeepCopy here is to satisfy the requirement of gosec
-		require.NoError(t, c.Delete(ctx, e.DeepCopy()))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-
-	var httproutes gwapiv1.HTTPRouteList
-	err = c.List(ctx, &httproutes)
-	require.NoError(t, err)
-	for _, e := range httproutes.Items {
-		require.NoError(t, c.Delete(ctx, e.DeepCopy()))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-
-	var virtualservices istiov1b1.VirtualServiceList
-	err = c.List(ctx, &virtualservices)
-	require.NoError(t, err)
-	for _, e := range virtualservices.Items {
-		require.NoError(t, c.Delete(ctx, e))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-	// let HTNN to clean up EnvoyFilter
+	k8s.CleanUp(t, suite.Opt.Client)
 }
 
 func (suite *Suite) K8sClient() client.Client {

go.mod

@@ -9,6 +9,7 @@ toolchain go1.21.5
 
 require (
 	github.com/agiledragon/gomonkey/v2 v2.11.0
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/casbin/casbin/v2 v2.82.0
 	github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101
 	github.com/coreos/go-oidc/v3 v3.9.0
@@ -30,9 +31,9 @@ require (
 	github.com/redis/go-redis/v9 v9.5.1
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.8.4
-	go.uber.org/zap v1.26.0
-	golang.org/x/net v0.19.0
-	golang.org/x/oauth2 v0.15.0
+	go.uber.org/zap v1.27.0
+	golang.org/x/net v0.21.0
+	golang.org/x/oauth2 v0.17.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.5.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17
@@ -119,11 +120,11 @@ require (
 	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect
 	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect