test(api): add resource deletion step in e2e test cases

- exclude namespace in resource deletion step; - refactor `apply` and `delete` kubectl commands; - handle errors in clean up phase; - add resource deletion step in each test case. Signed-off-by: Roman Sysoev <[email protected]>
deckhouse · Nov 11, 2024 · bde4c17 · bde4c17
1 parent 394e461
commit bde4c17
Show file tree

Hide file tree

Showing 15 changed files with 399 additions and 100 deletions.
diff --git a/tests/e2e/complex_test.go b/tests/e2e/complex_test.go
@@ -67,7 +67,10 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 
 	Context("When virtualization resources are applied:", func() {
 		It("result should be succeeded", func() {
-			res := kubectl.Kustomize(conf.TestData.ComplexTest, kc.KustomizeOptions{})
+			res := kubectl.Apply(kc.ApplyOptions{
+				Filename:       []string{conf.TestData.ComplexTest},
+				FilenameOption: kc.Kustomize,
+			})
 			Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr())
 		})
 	})
@@ -225,5 +228,24 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 				CheckExternalConnection(externalHost, httpStatusOk, vms...)
 			})
 		})
+
+		Context("When test is complited:", func() {
+			It("tries to delete used resources", func() {
+				kustimizationFile := fmt.Sprintf("%s/%s", conf.TestData.ComplexTest, "kustomization.yaml")
+				err := kustomize.ExcludeResource(kustimizationFile, "ns.yaml")
+				Expect(err).NotTo(HaveOccurred(), "cannot exclude namespace from clean up operation:\n%s", err)
+				res := kubectl.Delete(kc.DeleteOptions{
+					Filename:       []string{conf.TestData.ComplexTest},
+					FilenameOption: kc.Kustomize,
+				})
+				Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr())
+				res = kubectl.Delete(kc.DeleteOptions{
+					Labels:    testCaseLabel,
+					Namespace: conf.Namespace,
+					Resource:  kc.ResourceKubevirtVMIM,
+				})
+				Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr())
+			})
+		})
 	})
 })
diff --git a/tests/e2e/config/config.go b/tests/e2e/config/config.go
@@ -263,6 +263,39 @@ func (k *Kustomize) SetParams(filePath, namespace, namePrefix string) error {
 	return nil
 }
 
+func (k *Kustomize) ExcludeResource(filePath, resourceName string) error {
+	var kustomizeFile Kustomize
+
+	data, readErr := os.ReadFile(filePath)
+	if readErr != nil {
+		return readErr
+	}
+
+	unmarshalErr := yamlv3.Unmarshal([]byte(data), &kustomizeFile)
+	if unmarshalErr != nil {
+		return unmarshalErr
+	}
+	newResourceList := make([]string, 0, len(kustomizeFile.Resources))
+	for _, v := range kustomizeFile.Resources {
+		if v != resourceName {
+			newResourceList = append(newResourceList, v)
+		}
+	}
+
+	kustomizeFile.Resources = newResourceList
+	updatedKustomizeFile, marshalErr := yamlv3.Marshal(&kustomizeFile)
+	if marshalErr != nil {
+		return marshalErr
+	}
+
+	writeErr := os.WriteFile(filePath, updatedKustomizeFile, 0o644)
+	if writeErr != nil {
+		return writeErr
+	}
+
+	return nil
+}
+
 func GetModuleConfig() (*ModuleConfig, error) {
 	res := kubectl.GetResource(kc.ResourceModuleConfig, "virtualization", kc.GetOptions{Output: "yaml"})
 	if !res.WasSuccess() {

diff --git a/tests/e2e/disks_test.go b/tests/e2e/disks_test.go
@@ -75,7 +75,10 @@ var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 	Context("CVI", func() {
 		AfterAll(func() {
 			By("Removing resources for cvi tests")
-			kubectl.Delete(conf.Disks.CviTestDataDir, kc.DeleteOptions{})
+			kubectl.Delete(kc.DeleteOptions{
+				Filename:       []string{conf.Disks.CviTestDataDir},
+				FilenameOption: kc.Filename,
+			})
 		})
 		When("http source", func() {
 			filepath := cviPath("/cvi_http.yaml")
@@ -104,8 +107,10 @@ var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 		When("upload", func() {
 			AfterAll(func() {
 				By("Removing support resources for cvi upload test")
-				kubectl.DeleteResource(kc.ResourcePod, UploadHelpPod, kc.DeleteOptions{
+				kubectl.Delete(kc.DeleteOptions{
+					Filename:  []string{UploadHelpPod},
 					Namespace: conf.Namespace,
+					Resource:  kc.ResourcePod,
 				})
 			})
 			filepath := cviPath("/cvi_upload.yaml")
@@ -116,7 +121,10 @@ var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 	Context("VI", func() {
 		AfterAll(func() {
 			By("Removing resources for vi tests")
-			kubectl.Delete(conf.Disks.ViTestDataDir, kc.DeleteOptions{})
+			kubectl.Delete(kc.DeleteOptions{
+				Filename:       []string{conf.Disks.ViTestDataDir},
+				FilenameOption: kc.Filename,
+			})
 		})
 		When("http source", func() {
 			filepath := viPath("/vi_http.yaml")
@@ -145,8 +153,10 @@ var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 		When("upload", func() {
 			AfterAll(func() {
 				By("Removing support resources for vi upload test")
-				kubectl.DeleteResource(kc.ResourcePod, UploadHelpPod, kc.DeleteOptions{
+				kubectl.Delete(kc.DeleteOptions{
+					Filename:  []string{UploadHelpPod},
 					Namespace: conf.Namespace,
+					Resource:  kc.ResourcePod,
 				})
 			})
 			filepath := viPath("/vi_upload.yaml")
@@ -157,7 +167,10 @@ var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 	Context("VD", func() {
 		AfterAll(func() {
 			By("Removing resources for vd tests")
-			kubectl.Delete(conf.Disks.VdTestDataDir, kc.DeleteOptions{})
+			kubectl.Delete(kc.DeleteOptions{
+				Filename:       []string{conf.Disks.VdTestDataDir},
+				FilenameOption: kc.Filename,
+			})
 		})
 		When("http source", func() {
 			filepath := vdPath("/vd_http.yaml")
@@ -197,8 +210,10 @@ var _ = Describe("Disks", ginkgoutil.CommonE2ETestDecorators(), func() {
 		When("upload", func() {
 			AfterAll(func() {
 				By("Removing support resources for vd upload test")
-				kubectl.DeleteResource(kc.ResourcePod, UploadHelpPod, kc.DeleteOptions{
+				kubectl.Delete(kc.DeleteOptions{
+					Filename:  []string{UploadHelpPod},
 					Namespace: conf.Namespace,
+					Resource:  kc.ResourcePod,
 				})
 			})
 			filepath := vdPath("/vd_upload.yaml")

diff --git a/tests/e2e/ipam_test.go b/tests/e2e/ipam_test.go
@@ -34,15 +34,21 @@ var _ = Describe("Ipam", func() {
 	Context("VirtualMachineIPAddressClaim", ginkgoutil.CommonE2ETestDecorators(), func() {
 		AfterAll(func() {
 			By("Removing resources for vmip tests")
-			kubectl.Delete(conf.Ipam.TestDataDir, kc.DeleteOptions{})
+			kubectl.Delete(kc.DeleteOptions{
+				Filename:       []string{conf.Ipam.TestDataDir},
+				FilenameOption: kc.Filename,
+			})
 		})
 		GetLeaseNameFromClaim := func(manifestClaim string) string {
 			res := kubectl.Get(manifestClaim, kc.GetOptions{Output: "jsonpath={.spec.virtualMachineIPAddressLeaseName}"})
 			Expect(res.Error()).NotTo(HaveOccurred(), "failed get vmip from file %s.\n%s", manifestClaim, res.StdErr())
 			return res.StdOut()
 		}
 		DeleteVMIP := func(manifest string) {
-			res := kubectl.Delete(manifest, kc.DeleteOptions{})
+			res := kubectl.Delete(kc.DeleteOptions{
+				Filename:       []string{manifest},
+				FilenameOption: kc.Filename,
+			})
 			Expect(res.Error()).NotTo(HaveOccurred(), "failed delete vmip from file %s.\n%s", manifest, res.StdErr())
 		}
 		When("reclaimPolicy Delete", func() {
@@ -65,7 +71,11 @@ var _ = Describe("Ipam", func() {
 				CheckField(kc.ResourceVMIPLease, leaseName, "jsonpath={'.status.phase'}", PhaseBound)
 				DeleteVMIP(filepath)
 				CheckField(kc.ResourceVMIPLease, leaseName, "jsonpath={'.status.phase'}", PhaseReleased)
-				kubectl.DeleteResource(kc.ResourceVMIPLease, leaseName, kc.DeleteOptions{Namespace: conf.Namespace})
+				kubectl.Delete(kc.DeleteOptions{
+					Filename:  []string{leaseName},
+					Namespace: conf.Namespace,
+					Resource:  kc.ResourceVMIPLease,
+				})
 			})
 		})
 	})

diff --git a/tests/e2e/kubectl/kubectl.go b/tests/e2e/kubectl/kubectl.go
@@ -29,24 +29,27 @@ import (
 )
 
 const (
-	Cmd              = "kubectl"
-	ShortTimeout     = 10 * time.Second
-	MediumTimeout    = 30 * time.Second
-	LongTimeout      = 60 * time.Second
-	ExecExtraTimeout = 20 * time.Second
+	Cmd                             = "kubectl"
+	ShortTimeout                    = 10 * time.Second
+	MediumTimeout                   = 30 * time.Second
+	LongTimeout                     = 60 * time.Second
+	ExecExtraTimeout                = 20 * time.Second
+	Filename         FilenameOption = "--filename"
+	Kustomize        FilenameOption = "--kustomize"
 )
 
-type Resource string
+type (
+	Resource       string
+	FilenameOption string
+)
 
 type Kubectl interface {
-	Apply(filepath string, opts ApplyOptions) *executor.CMDResult
+	Apply(opts ApplyOptions) *executor.CMDResult
 	Create(filepath string, opts CreateOptions) *executor.CMDResult
 	CreateResource(resource Resource, name string, opts CreateOptions) *executor.CMDResult
 	Get(filepath string, opts GetOptions) *executor.CMDResult
 	GetResource(resource Resource, name string, opts GetOptions) *executor.CMDResult
-	Delete(filepath string, opts DeleteOptions) *executor.CMDResult
-	DeleteResource(resource Resource, name string, opts DeleteOptions) *executor.CMDResult
-	Kustomize(directory string, opts KustomizeOptions) *executor.CMDResult
+	Delete(opts DeleteOptions) *executor.CMDResult
 	List(resource Resource, opts GetOptions) *executor.CMDResult
 	Wait(filepath string, opts WaitOptions) *executor.CMDResult
 	WaitResource(resource Resource, name string, opts WaitOptions) *executor.CMDResult
@@ -56,10 +59,17 @@ type Kubectl interface {
 	RawCommand(subCmd string, timeout time.Duration) *executor.CMDResult
 }
 
+// FilenameOption:
+//
+//	kubectl.Filename  // --filename
+//	kubectl.Kustomize // --kustomize
 type ApplyOptions struct {
-	Namespace string
-	Output    string
-	Force     bool
+	Filename       []string
+	FilenameOption FilenameOption
+	Force          bool
+	Namespace      string
+	Output         string
+	Recursive      bool
 }
 
 type CreateOptions struct {
@@ -69,8 +79,13 @@ type CreateOptions struct {
 
 type DeleteOptions struct {
 	ExcludedLabels []string
+	Filename       []string
+	FilenameOption FilenameOption
+	IgnoreNotFound bool
 	Labels         map[string]string
 	Namespace      string
+	Recursive      bool
+	Resource       Resource
 }
 
 type GetOptions struct {
@@ -81,12 +96,6 @@ type GetOptions struct {
 	Output         string
 }
 
-type KustomizeOptions struct {
-	Namespace string
-	Output    string
-	Force     bool
-}
-
 type WaitOptions struct {
 	ExcludedLabels []string
 	Labels         map[string]string
@@ -154,10 +163,10 @@ type KubectlCMD struct {
 	cmd string
 }
 
-func (k KubectlCMD) Apply(filepath string, opts ApplyOptions) *executor.CMDResult {
-	cmd := fmt.Sprintf("%s apply -f %s", k.cmd, filepath)
+func (k KubectlCMD) Apply(opts ApplyOptions) *executor.CMDResult {
+	cmd := fmt.Sprintf("%s apply", k.cmd)
 	cmd = k.applyOptions(cmd, opts)
-	ctx, cancel := context.WithTimeout(context.Background(), ShortTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), MediumTimeout)
 	defer cancel()
 	return k.ExecContext(ctx, cmd)
 }
@@ -194,26 +203,12 @@ func (k KubectlCMD) GetResource(resource Resource, name string, opts GetOptions)
 	return k.ExecContext(ctx, cmd)
 }
 
-func (k KubectlCMD) Delete(filepath string, opts DeleteOptions) *executor.CMDResult {
-	cmd := fmt.Sprintf("%s delete -f %s", k.cmd, filepath)
-	cmd = k.deleteOptions(cmd, opts)
-	return k.Exec(cmd)
-}
-
-func (k KubectlCMD) DeleteResource(resource Resource, name string, opts DeleteOptions) *executor.CMDResult {
-	cmd := fmt.Sprintf("%s delete %s %s", k.cmd, resource, name)
+func (k KubectlCMD) Delete(opts DeleteOptions) *executor.CMDResult {
+	cmd := fmt.Sprintf("%s delete", k.cmd)
 	cmd = k.deleteOptions(cmd, opts)
 	return k.Exec(cmd)
 }
 
-func (k KubectlCMD) Kustomize(directory string, opts KustomizeOptions) *executor.CMDResult {
-	cmd := fmt.Sprintf("%s apply --kustomize %s", k.cmd, directory)
-	cmd = k.kustomizeOptions(cmd, opts)
-	ctx, cancel := context.WithTimeout(context.Background(), LongTimeout)
-	defer cancel()
-	return k.ExecContext(ctx, cmd)
-}
-
 func (k KubectlCMD) List(resource Resource, opts GetOptions) *executor.CMDResult {
 	cmd := fmt.Sprintf("%s get %s", k.cmd, resource)
 	cmd = k.getOptions(cmd, opts)
@@ -283,6 +278,23 @@ func (k KubectlCMD) addNamespace(cmd, ns string) string {
 	return cmd
 }
 
+func (k KubectlCMD) addFilenameOptions(cmd string, resource Resource, filenameOpt FilenameOption, recursive bool, filenames ...string) string {
+	if resource != "" {
+		cmd = fmt.Sprintf("%s %s", cmd, resource)
+	}
+	if filenameOpt != "" {
+		cmd = fmt.Sprintf("%s %s", cmd, filenameOpt)
+	}
+	if len(filenames) != 0 {
+		files := strings.Join(filenames, " ")
+		cmd = fmt.Sprintf("%s %s", cmd, files)
+	}
+	if recursive {
+		cmd = fmt.Sprintf("%s --recoursive=%t", cmd, recursive)
+	}
+	return cmd
+}
+
 func (k KubectlCMD) addLabels(cmd string, labels map[string]string, excludedLabels []string) string {
 	if len(labels) != 0 || len(excludedLabels) != 0 {
 		rawLabels := make([]string, 0, len(labels)+len(excludedLabels))
@@ -316,12 +328,8 @@ func (k KubectlCMD) addIgnoreNotFound(cmd string, ignoreNotFound bool) string {
 }
 
 func (k KubectlCMD) applyOptions(cmd string, opts ApplyOptions) string {
-	cmd = k.addNamespace(cmd, opts.Namespace)
-	cmd = k.addOutput(cmd, opts.Output)
-	return fmt.Sprintf("%s --force=%t", cmd, opts.Force)
-}
-
-func (k KubectlCMD) kustomizeOptions(cmd string, opts KustomizeOptions) string {
+	var resourceEmptyValue Resource = ""
+	cmd = k.addFilenameOptions(cmd, resourceEmptyValue, opts.FilenameOption, opts.Recursive, opts.Filename...)
 	cmd = k.addNamespace(cmd, opts.Namespace)
 	cmd = k.addOutput(cmd, opts.Output)
 	return fmt.Sprintf("%s --force=%t", cmd, opts.Force)
@@ -342,8 +350,10 @@ func (k KubectlCMD) getOptions(cmd string, opts GetOptions) string {
 }
 
 func (k KubectlCMD) deleteOptions(cmd string, opts DeleteOptions) string {
+	cmd = k.addFilenameOptions(cmd, opts.Resource, opts.FilenameOption, opts.Recursive, opts.Filename...)
 	cmd = k.addNamespace(cmd, opts.Namespace)
 	cmd = k.addLabels(cmd, opts.Labels, opts.ExcludedLabels)
+	cmd = k.addIgnoreNotFound(cmd, opts.IgnoreNotFound)
 	return cmd
 }