From af4a310a5cc536155597fbc04068215790aae977 Mon Sep 17 00:00:00 2001 From: LavenderQAQ Date: Wed, 25 Oct 2023 02:24:45 +0000 Subject: [PATCH] test: added e2e test for iot systems Signed-off-by: LavenderQAQ --- test/e2e/cmd/init/init.go | 5 ++ test/e2e/e2e_test.go | 3 + test/e2e/util/nodepool.go | 38 ++++++++++ test/e2e/util/util.go | 4 ++ test/e2e/yurt/iot.go | 142 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+) create mode 100644 test/e2e/yurt/iot.go diff --git a/test/e2e/cmd/init/init.go b/test/e2e/cmd/init/init.go index 8e4f7753835..6b3c7066930 100644 --- a/test/e2e/cmd/init/init.go +++ b/test/e2e/cmd/init/init.go @@ -78,6 +78,7 @@ var ( yurtHubImageFormat = "openyurt/yurthub:%s" yurtManagerImageFormat = "openyurt/yurt-manager:%s" nodeServantImageFormat = "openyurt/node-servant:%s" + yurtIotDockImageFormat = "openyurt/yurt-iot-dock:%s" ) func NewInitCMD(out io.Writer) *cobra.Command { @@ -192,6 +193,7 @@ func (o *kindOptions) Config() *initializerConfig { YurtHubImage: fmt.Sprintf(yurtHubImageFormat, o.OpenYurtVersion), YurtManagerImage: fmt.Sprintf(yurtManagerImageFormat, o.OpenYurtVersion), NodeServantImage: fmt.Sprintf(nodeServantImageFormat, o.OpenYurtVersion), + yurtIotDockImage: fmt.Sprintf(yurtIotDockImageFormat, o.OpenYurtVersion), EnableDummyIf: o.EnableDummyIf, DisableDefaultCNI: o.DisableDefaultCNI, } @@ -237,6 +239,7 @@ type initializerConfig struct { YurtHubImage string YurtManagerImage string NodeServantImage string + yurtIotDockImage string EnableDummyIf bool DisableDefaultCNI bool } @@ -313,6 +316,7 @@ func (ki *Initializer) prepareImages() error { ki.YurtHubImage, ki.YurtManagerImage, ki.NodeServantImage, + ki.yurtIotDockImage, }, ki.CloudNodes); err != nil { return err } @@ -321,6 +325,7 @@ func (ki *Initializer) prepareImages() error { if err := ki.loadImagesToKindNodes([]string{ ki.YurtHubImage, ki.NodeServantImage, + ki.yurtIotDockImage, }, ki.EdgeNodes); err != nil { return err } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 5c9ee407a53..a8d68bc03b7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -74,6 +74,9 @@ var _ = ginkgo.BeforeSuite(func() { _, err = ns.CreateNameSpace(c, constants.YurtE2ENamespaceName) gomega.Expect(err).NotTo(gomega.HaveOccurred(), "fail to create namespace") + err = util.PrepareNodePoolWithNode(context.TODO(), yurtconfig.YurtE2eCfg.RuntimeClient, "openyurt-e2e-test-worker") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "fail to create a nodepool with node") + if labelFilter([]string{"edge-autonomy"}) { // get nginx podIP on edge node worker2 cs := c diff --git a/test/e2e/util/nodepool.go b/test/e2e/util/nodepool.go index 0830f40cd1c..b63cfb0a124 100644 --- a/test/e2e/util/nodepool.go +++ b/test/e2e/util/nodepool.go @@ -20,6 +20,7 @@ import ( "context" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" @@ -111,3 +112,40 @@ func InitNodeAndNodePool(ctx context.Context, k8sClient client.Client, poolToNod } return nil } + +const ( + NodePoolName = "nodepool-with-node" +) + +// PrepareNodePoolWithNode will create a edge nodepool named "nodepool-with-node" and add the "openyurt-e2e-test-worker" node to this nodepool. +// In order for Pods to be successfully deployed in e2e tests, a nodepool with nodes needs to be created +func PrepareNodePoolWithNode(ctx context.Context, k8sClient client.Client, nodeName string) error { + if err := k8sClient.Get(ctx, client.ObjectKey{Name: NodePoolName}, &v1beta1.NodePool{}); err == nil { + return nil + } else if !errors.IsNotFound(err) { + return err + } + + if err := k8sClient.Create(ctx, &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: NodePoolName, + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + }}); err != nil { + return err + } + + node := &corev1.Node{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, node); err != nil { + return err + } + + patchObj := client.MergeFrom(node.DeepCopy()) + node.Labels[apps.NodePoolLabel] = NodePoolName + + if err := k8sClient.Patch(ctx, node, patchObj); err != nil { + return err + } + return nil +} diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index d64a3a329b7..f92ef512a31 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -35,6 +35,8 @@ import ( appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" "github.com/openyurtio/openyurt/test/e2e/yurtconfig" ) @@ -46,6 +48,8 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(appsv1alpha1.AddToScheme(scheme)) utilruntime.Must(appsv1beta1.AddToScheme(scheme)) + utilruntime.Must(iotv1alpha1.AddToScheme(scheme)) + utilruntime.Must(iotv1alpha2.AddToScheme(scheme)) } const ( diff --git a/test/e2e/yurt/iot.go b/test/e2e/yurt/iot.go new file mode 100644 index 00000000000..3a0b2b00230 --- /dev/null +++ b/test/e2e/yurt/iot.go @@ -0,0 +1,142 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurt + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" + "github.com/openyurtio/openyurt/test/e2e/util" + ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" +) + +func generateTestVersions() []string { + return []string{"levski", "jakarta", "kamakura", "ireland", "minnesota"} +} + +var _ = Describe("OpenYurt IoT Test", func() { + var platformAdminName string + + ctx := context.Background() + k8sClient := ycfg.YurtE2eCfg.RuntimeClient + timeout := 60 * time.Second + platformadminTimeout := 5 * time.Minute + testVersions := generateTestVersions() + namespaceName := "iot-test-namespace" + nodePoolName := util.NodePoolName + + createNamespace := func() { + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + Eventually( + func() error { + return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) + }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + By("make sure the needed namespace are removed") + + res := &corev1.Namespace{} + Eventually( + func() error { + return k8sClient.Get(ctx, client.ObjectKey{ + Name: namespaceName, + }, res) + }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(&util.NotFoundMatcher{}) + Eventually( + func() error { + return k8sClient.Create(ctx, &ns) + }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + + createPlatformAdmin := func(version string) { + Eventually(func() error { + return k8sClient.Delete(ctx, &iotv1alpha2.PlatformAdmin{ + ObjectMeta: metav1.ObjectMeta{ + Name: platformAdminName, + Namespace: namespaceName, + }, + }) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + + testPlatformAdmin := iotv1alpha2.PlatformAdmin{ + ObjectMeta: metav1.ObjectMeta{ + Name: platformAdminName, + Namespace: namespaceName, + }, + Spec: iotv1alpha2.PlatformAdminSpec{ + Version: version, + PoolName: nodePoolName, + }, + } + Eventually(func() error { + return k8sClient.Create(ctx, &testPlatformAdmin) + }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + } + + BeforeEach(func() { + By("Start to run iot test, clean up previous resources") + k8sClient = ycfg.YurtE2eCfg.RuntimeClient + createNamespace() + }) + + AfterEach(func() { + By("Cleanup resources after test") + By(fmt.Sprintf("Delete the entire namespaceName %s", namespaceName)) + Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(BeNil()) + }) + + for _, testVersion := range testVersions { + version := testVersion + Describe(fmt.Sprintf("Test the %s version of PlatformAdmin", version), func() { + BeforeEach(func() { + platformAdminName = "test-platform-admin-" + version + createPlatformAdmin(version) + }) + + AfterEach(func() { + By(fmt.Sprintf("Delete the platformAdmin %s", platformAdminName)) + Expect(k8sClient.Delete(ctx, &iotv1alpha2.PlatformAdmin{ObjectMeta: metav1.ObjectMeta{Name: platformAdminName, Namespace: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(BeNil()) + }) + + It(fmt.Sprintf("The %s version of PlatformAdmin should be stable in ready state after it is created", version), func() { + By("verify the status of platformadmin") + Eventually(func() error { + testPlatfromAdmin := &iotv1alpha2.PlatformAdmin{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: platformAdminName, Namespace: namespaceName}, testPlatfromAdmin); err != nil { + return err + } + if testPlatfromAdmin.Status.Ready == true { + return nil + } else { + return fmt.Errorf("The %s version of PlatformAdmin is not ready", version) + } + }, platformadminTimeout, 5*time.Second).Should(Succeed()) + }) + }) + } +})