make test stable

controller/tests/benchmark/httpfilterpolicy_controller_test.go

@@ -79,21 +79,21 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 		var policies mosniov1.HTTPFilterPolicyList
 		if err := k8sClient.List(ctx, &policies); err == nil {
 			for _, e := range policies.Items {
-				Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, &e)
 			}
 		}
 
 		var virtualservices istiov1b1.VirtualServiceList
 		if err := k8sClient.List(ctx, &virtualservices); err == nil {
 			for _, e := range virtualservices.Items {
-				Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, e)
 			}
 		}
 
 		var gateways istiov1b1.GatewayList
 		if err := k8sClient.List(ctx, &gateways); err == nil {
 			for _, e := range gateways.Items {
-				Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, e)
 			}
 		}
 	})

controller/tests/integration/controller/consumer_controller_test.go

@@ -50,14 +50,14 @@ var _ = Describe("Consumer controller", func() {
 			var consumers mosniov1.ConsumerList
 			if err := k8sClient.List(ctx, &consumers); err == nil {
 				for _, e := range consumers.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var envoyfilters istiov1a3.EnvoyFilterList
 			if err := k8sClient.List(ctx, &envoyfilters); err == nil {
 				for _, e := range envoyfilters.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 		})

controller/tests/integration/controller/httpfilterpolicy_controller_test.go

@@ -76,7 +76,7 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 			var policies mosniov1.HTTPFilterPolicyList
 			if err := k8sClient.List(ctx, &policies); err == nil {
 				for _, e := range policies.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 		})
@@ -159,28 +159,28 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 			var policies mosniov1.HTTPFilterPolicyList
 			if err := k8sClient.List(ctx, &policies); err == nil {
 				for _, e := range policies.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var virtualservices istiov1b1.VirtualServiceList
 			if err := k8sClient.List(ctx, &virtualservices); err == nil {
 				for _, e := range virtualservices.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 
 			var gateways istiov1b1.GatewayList
 			if err := k8sClient.List(ctx, &gateways); err == nil {
 				for _, e := range gateways.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 
 			var envoyfilters istiov1a3.EnvoyFilterList
 			if err := k8sClient.List(ctx, &envoyfilters); err == nil {
 				for _, e := range envoyfilters.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 
@@ -585,28 +585,28 @@ var _ = Describe("HTTPFilterPolicy controller", func() {
 			var policies mosniov1.HTTPFilterPolicyList
 			if err := k8sClient.List(ctx, &policies); err == nil {
 				for _, e := range policies.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var httproutes gwapiv1.HTTPRouteList
 			if err := k8sClient.List(ctx, &httproutes); err == nil {
 				for _, e := range httproutes.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var gateways gwapiv1.GatewayList
 			if err := k8sClient.List(ctx, &gateways); err == nil {
 				for _, e := range gateways.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 
 			var envoyfilters istiov1a3.EnvoyFilterList
 			if err := k8sClient.List(ctx, &envoyfilters); err == nil {
 				for _, e := range envoyfilters.Items {
-					Expect(k8sClient.Delete(ctx, e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, e)
 				}
 			}
 

controller/tests/integration/controller/serviceregistry_controller_test.go

@@ -46,7 +46,7 @@ var _ = Describe("ServiceRegistry controller", func() {
 			var registries mosniov1.ServiceRegistryList
 			if err := k8sClient.List(ctx, &registries); err == nil {
 				for _, e := range registries.Items {
-					Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+					pkg.DeleteK8sResource(ctx, k8sClient, &e)
 				}
 			}
 

controller/tests/integration/registries/nacos_test.go

@@ -138,7 +138,7 @@ var _ = Describe("Nacos", func() {
 		var registries mosniov1.ServiceRegistryList
 		if err := k8sClient.List(ctx, &registries); err == nil {
 			for _, e := range registries.Items {
-				Expect(k8sClient.Delete(ctx, &e)).Should(Succeed())
+				pkg.DeleteK8sResource(ctx, k8sClient, &e)
 			}
 		}
 
@@ -242,7 +242,7 @@ var _ = Describe("Nacos", func() {
 		Expect(k8sClient.Patch(ctx, currNacos, base)).Should(Succeed())
 		Eventually(func() bool {
 			entries = listServiceEntries()
-			return len(entries) == 2
+			return len(entries) == 1
 		}, timeout, interval).Should(BeTrue())
 
 		// subscribe change

controller/tests/pkg/utils.go

@@ -15,13 +15,16 @@
 package pkg
 
 import (
+	"context"
 	"encoding/json"
 	"strings"
 	"testing"
 
+	. "github.com/onsi/gomega"
 	"github.com/stretchr/testify/require"
 	istiov1a3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
 	istiov1b1 "istio.io/client-go/pkg/apis/networking/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
@@ -74,3 +77,10 @@ func FakeK8sClient(t *testing.T) client.Client {
 	require.NoError(t, err)
 	return k8sClient
 }
+
+func DeleteK8sResource(ctx context.Context, k8sClient client.Client, obj client.Object, opts ...client.DeleteOption) {
+	err := k8sClient.Delete(ctx, obj, opts...)
+	if err != nil && !apierrors.IsNotFound(err) {
+		Expect(err).Should(Succeed())
+	}
+}

e2e/pkg/k8s/k8s.go

@@ -24,12 +24,15 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	istiov1b1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
+	mosniov1 "mosn.io/htnn/controller/api/v1"
 	"mosn.io/htnn/pkg/log"
 )
 
@@ -100,3 +103,43 @@ func readResources(decoder *yaml.YAMLOrJSONDecoder) ([]unstructured.Unstructured
 
 	return resources, nil
 }
+
+func deleteResource(t *testing.T, ctx context.Context, k8sClient client.Client, obj client.Object, opts ...client.DeleteOption) {
+	err := k8sClient.Delete(ctx, obj, opts...)
+	if err != nil && !apierrors.IsNotFound(err) {
+		require.NoError(t, err)
+	}
+	logger.Info("Deleted", "name", obj.GetName(), "kind", obj.GetObjectKind())
+}
+
+func CleanUp(t *testing.T, c client.Client) {
+	ctx := context.Background()
+	var policies mosniov1.HTTPFilterPolicyList
+	err := c.List(ctx, &policies)
+	require.NoError(t, err)
+	for _, e := range policies.Items {
+		deleteResource(t, ctx, c, &e)
+	}
+
+	var consumers mosniov1.ConsumerList
+	err = c.List(ctx, &consumers)
+	require.NoError(t, err)
+	for _, e := range consumers.Items {
+		deleteResource(t, ctx, c, &e)
+	}
+
+	var httproutes gwapiv1.HTTPRouteList
+	err = c.List(ctx, &httproutes)
+	require.NoError(t, err)
+	for _, e := range httproutes.Items {
+		deleteResource(t, ctx, c, &e)
+	}
+
+	var virtualservices istiov1b1.VirtualServiceList
+	err = c.List(ctx, &virtualservices)
+	require.NoError(t, err)
+	for _, e := range virtualservices.Items {
+		deleteResource(t, ctx, c, e)
+	}
+	// let HTNN to clean up EnvoyFilter
+}

e2e/pkg/suite/suite.go

@@ -31,13 +31,10 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-	istiov1b1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
 
-	mosniov1 "mosn.io/htnn/controller/api/v1"
 	"mosn.io/htnn/e2e/pkg/k8s"
 	"mosn.io/htnn/pkg/log"
 )
@@ -145,42 +142,7 @@ func (suite *Suite) stopPortForward(t *testing.T) {
 }
 
 func (suite *Suite) cleanup(t *testing.T) {
-	c := suite.Opt.Client
-	ctx := context.Background()
-	var policies mosniov1.HTTPFilterPolicyList
-	err := c.List(ctx, &policies)
-	require.NoError(t, err)
-	for _, e := range policies.Items {
-		// DeepCopy here is to satisfy the requirement of gosec
-		require.NoError(t, c.Delete(ctx, e.DeepCopy()))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-
-	var consumers mosniov1.ConsumerList
-	err = c.List(ctx, &consumers)
-	require.NoError(t, err)
-	for _, e := range consumers.Items {
-		// DeepCopy here is to satisfy the requirement of gosec
-		require.NoError(t, c.Delete(ctx, e.DeepCopy()))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-
-	var httproutes gwapiv1.HTTPRouteList
-	err = c.List(ctx, &httproutes)
-	require.NoError(t, err)
-	for _, e := range httproutes.Items {
-		require.NoError(t, c.Delete(ctx, e.DeepCopy()))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-
-	var virtualservices istiov1b1.VirtualServiceList
-	err = c.List(ctx, &virtualservices)
-	require.NoError(t, err)
-	for _, e := range virtualservices.Items {
-		require.NoError(t, c.Delete(ctx, e))
-		logger.Info("Deleted", "name", e.GetName(), "kind", e.GetObjectKind())
-	}
-	// let HTNN to clean up EnvoyFilter
+	k8s.CleanUp(t, suite.Opt.Client)
 }
 
 func (suite *Suite) K8sClient() client.Client {

go.mod

@@ -9,6 +9,7 @@ toolchain go1.21.5
 
 require (
 	github.com/agiledragon/gomonkey/v2 v2.11.0
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/casbin/casbin/v2 v2.82.0
 	github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101
 	github.com/coreos/go-oidc/v3 v3.9.0

go.sum

@@ -12,6 +12,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=

plugins/oidc/config.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/avast/retry-go"
 	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/gorilla/securecookie"
 	"golang.org/x/oauth2"
@@ -72,7 +73,21 @@ func ctxWithClient(ctx context.Context) context.Context {
 
 func (conf *config) Init(cb api.ConfigCallbackHandler) error {
 	ctx := ctxWithClient(context.Background())
-	provider, err := oidc.NewProvider(ctx, conf.Issuer)
+	var provider *oidc.Provider
+	var err error
+	err = retry.Do(
+		func() error {
+			provider, err = oidc.NewProvider(ctx, conf.Issuer)
+			return err
+		},
+		retry.RetryIf(func(err error) bool {
+			api.LogWarnf("failed to get oidc provider, err: %v", err)
+			return true
+		}),
+		retry.Attempts(3),
+		// backoff delay
+		retry.Delay(500*time.Millisecond),
+	)
 	if err != nil {
 		return err
 	}

plugins/tests/integration/oidc_test.go

@@ -79,7 +79,7 @@ func TestOIDC(t *testing.T) {
 		require.Nil(t, err)
 		uri = resp.Header.Get("Location")
 		return uri != ""
-	}, 5*time.Second, 1*time.Second)
+	}, 10*time.Second, 1*time.Second)
 
 	u, err := url.ParseRequestURI(uri)
 	require.NoError(t, err)