From f6ec2061da047741cdb3b36805451ac1ff0d0fa0 Mon Sep 17 00:00:00 2001 From: zzxwill Date: Sun, 21 Nov 2021 00:41:39 +0800 Subject: [PATCH 1/2] Add more e2e tests Added more tests to check Configuration --- controllers/client/init.go | 16 +++++ controllers/terraform/logging.go | 9 --- controllers/terraform/status.go | 4 +- e2e/configuration_test.go | 67 +++++++++++++++++++ .../alibaba_test.go | 2 +- .../suite_test.go | 2 +- 6 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 controllers/client/init.go create mode 100644 e2e/configuration_test.go rename e2e/{cloud-providers => configurations}/alibaba_test.go (98%) rename e2e/{cloud-providers => configurations}/suite_test.go (97%) diff --git a/controllers/client/init.go b/controllers/client/init.go new file mode 100644 index 00000000..ee57ab0f --- /dev/null +++ b/controllers/client/init.go @@ -0,0 +1,16 @@ +package client + +import ( + "k8s.io/client-go/kubernetes" + config2 "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +// InitClientSet initializes the Go client set +func InitClientSet() (*kubernetes.Clientset, error) { + config, err := config2.GetConfig() + if err != nil { + return nil, err + } + return kubernetes.NewForConfig(config) +} + diff --git a/controllers/terraform/logging.go b/controllers/terraform/logging.go index 68d0a325..ee3ed7e8 100644 --- a/controllers/terraform/logging.go +++ b/controllers/terraform/logging.go @@ -10,17 +10,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - config2 "sigs.k8s.io/controller-runtime/pkg/client/config" ) -func initClientSet() (*kubernetes.Clientset, error) { - config, err := config2.GetConfig() - if err != nil { - return nil, err - } - return kubernetes.NewForConfig(config) -} - func getPodLog(ctx context.Context, client *kubernetes.Clientset, namespace, jobName string) (string, error) { label := fmt.Sprintf("job-name=%s", jobName) pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: label}) diff --git a/controllers/terraform/status.go b/controllers/terraform/status.go index 77c10f1a..a4245ace 100644 --- a/controllers/terraform/status.go +++ b/controllers/terraform/status.go @@ -6,12 +6,14 @@ import ( "github.com/pkg/errors" "k8s.io/klog/v2" + + "github.com/oam-dev/terraform-controller/controllers/client" ) // GetTerraformStatus will get Terraform execution status func GetTerraformStatus(ctx context.Context, namespace, jobName string) error { klog.InfoS("checking Terraform execution status", "Namespace", namespace, "Job", jobName) - clientSet, err := initClientSet() + clientSet, err := client.InitClientSet() if err != nil { klog.ErrorS(err, "failed to init clientSet") return err diff --git a/e2e/configuration_test.go b/e2e/configuration_test.go new file mode 100644 index 00000000..9a7512bc --- /dev/null +++ b/e2e/configuration_test.go @@ -0,0 +1,67 @@ +package e2e + +import ( + "fmt" + "golang.org/x/net/context" + "gotest.tools/assert" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/oam-dev/terraform-controller/controllers/client" +) + +func TestConfigurationResult(t *testing.T) { + clientSet, err := client.InitClientSet() + assert.NilError(t, err) + ctx := context.Background() + + klog.Info("Applying Configuration") + pwd, _ := os.Getwd() + cmd := fmt.Sprintf("kubectl apply -f %s", filepath.Join(pwd, "..", "..", "examples/alibaba/eip/configuration_eip.yaml")) + err = exec.Command("bash", "-c", cmd).Start() + assert.NilError(t, err) + + klog.Info("Checking Configuration status") + for i := 0; i < 60; i++ { + var fields []string + output, err := exec.Command("bash", "-c", "kubectl get configuration").Output() + assert.NilError(t, err) + + lines := strings.Split(string(output), "\n") + for i, line := range lines { + if i == 0 { + continue + } + fields = strings.Fields(line) + if len(fields) == 0 { + continue + } + if !(len(fields) == 3 && fields[1] == "Available") { + break + } + } + time.Sleep(time.Second * 5) + } + + klog.Info("Checking Secret which stores variables") + _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "variable-alibaba-eip", v1.GetOptions{}) + assert.NilError(t, err) + + klog.Info("Checking Secret which stores outputs") + _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "eip-conn", v1.GetOptions{}) + assert.NilError(t, err) + + klog.Info("Checking Secret which stores Backend") + _, err = clientSet.CoreV1().Secrets("terraform").Get(ctx, "tfstate-default-alibaba-eip", v1.GetOptions{}) + assert.NilError(t, err) + + klog.Info("Checking ConfigMap which stores .tf") + _, err = clientSet.CoreV1().ConfigMaps("default").Get(ctx, "tf-alibaba-eip", v1.GetOptions{}) + assert.NilError(t, err) +} diff --git a/e2e/cloud-providers/alibaba_test.go b/e2e/configurations/alibaba_test.go similarity index 98% rename from e2e/cloud-providers/alibaba_test.go rename to e2e/configurations/alibaba_test.go index d1f37c9f..45d3730e 100644 --- a/e2e/cloud-providers/alibaba_test.go +++ b/e2e/configurations/alibaba_test.go @@ -1,4 +1,4 @@ -package e2e +package configurations import ( "fmt" diff --git a/e2e/cloud-providers/suite_test.go b/e2e/configurations/suite_test.go similarity index 97% rename from e2e/cloud-providers/suite_test.go rename to e2e/configurations/suite_test.go index fdf6054a..f2282c1e 100644 --- a/e2e/cloud-providers/suite_test.go +++ b/e2e/configurations/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package configurations import ( "testing" From fad4557e72360ff88e84752e009e45c376016782 Mon Sep 17 00:00:00 2001 From: zzxwill Date: Sun, 21 Nov 2021 12:43:27 +0800 Subject: [PATCH 2/2] Checking secrets and configmaps on applying and deleting Check whether configmaps and secrets and created when applying configurations. Check whetheter configmaps and secrets are deleted after deleting configurations --- controllers/client/init.go | 1 - e2e/configuration_test.go | 89 ++++++++++++++++++++++++------ e2e/configurations/alibaba_test.go | 1 - go.mod | 1 + 4 files changed, 74 insertions(+), 18 deletions(-) diff --git a/controllers/client/init.go b/controllers/client/init.go index ee57ab0f..5e9fbee7 100644 --- a/controllers/client/init.go +++ b/controllers/client/init.go @@ -13,4 +13,3 @@ func InitClientSet() (*kubernetes.Clientset, error) { } return kubernetes.NewForConfig(config) } - diff --git a/e2e/configuration_test.go b/e2e/configuration_test.go index 9a7512bc..9f1ddfb7 100644 --- a/e2e/configuration_test.go +++ b/e2e/configuration_test.go @@ -4,6 +4,7 @@ import ( "fmt" "golang.org/x/net/context" "gotest.tools/assert" + kerrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "os" @@ -16,18 +17,21 @@ import ( "github.com/oam-dev/terraform-controller/controllers/client" ) -func TestConfigurationResult(t *testing.T) { +const backendSecretNamespace = "vela-system" + +func TestConfiguration(t *testing.T) { clientSet, err := client.InitClientSet() assert.NilError(t, err) ctx := context.Background() - klog.Info("Applying Configuration") + klog.Info("1. Applying Configuration") pwd, _ := os.Getwd() - cmd := fmt.Sprintf("kubectl apply -f %s", filepath.Join(pwd, "..", "..", "examples/alibaba/eip/configuration_eip.yaml")) + configuration := filepath.Join(pwd, "..", "examples/alibaba/eip/configuration_eip.yaml") + cmd := fmt.Sprintf("kubectl apply -f %s", configuration) err = exec.Command("bash", "-c", cmd).Start() assert.NilError(t, err) - klog.Info("Checking Configuration status") + klog.Info("2. Checking Configuration status") for i := 0; i < 60; i++ { var fields []string output, err := exec.Command("bash", "-c", "kubectl get configuration").Output() @@ -39,29 +43,82 @@ func TestConfigurationResult(t *testing.T) { continue } fields = strings.Fields(line) - if len(fields) == 0 { - continue - } - if !(len(fields) == 3 && fields[1] == "Available") { - break + if len(fields) == 3 && fields[0] == "alibaba-eip" && fields[1] == "Available" { + goto continueCheck } } + if i == 59 { + t.Error("Configuration is not ready") + } time.Sleep(time.Second * 5) } - klog.Info("Checking Secret which stores variables") - _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "variable-alibaba-eip", v1.GetOptions{}) +continueCheck: + klog.Info("3. Checking Configuration status") + + klog.Info("- Checking ConfigMap which stores .tf") + _, err = clientSet.CoreV1().ConfigMaps("default").Get(ctx, "tf-alibaba-eip", v1.GetOptions{}) + assert.NilError(t, err) + + klog.Info("- Checking Secret which stores Backend") + _, err = clientSet.CoreV1().Secrets(backendSecretNamespace).Get(ctx, "tfstate-default-alibaba-eip", v1.GetOptions{}) assert.NilError(t, err) - klog.Info("Checking Secret which stores outputs") + klog.Info("- Checking Secret which stores outputs") _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "eip-conn", v1.GetOptions{}) assert.NilError(t, err) - klog.Info("Checking Secret which stores Backend") - _, err = clientSet.CoreV1().Secrets("terraform").Get(ctx, "tfstate-default-alibaba-eip", v1.GetOptions{}) + klog.Info("- Checking Secret which stores variables") + _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "variable-alibaba-eip", v1.GetOptions{}) assert.NilError(t, err) - klog.Info("Checking ConfigMap which stores .tf") - _, err = clientSet.CoreV1().ConfigMaps("default").Get(ctx, "tf-alibaba-eip", v1.GetOptions{}) + klog.Info("4. Deleting Configuration") + cmd = fmt.Sprintf("kubectl delete -f %s", configuration) + err = exec.Command("bash", "-c", cmd).Start() assert.NilError(t, err) + + klog.Info("5. Checking Configuration is deleted") + for i := 0; i < 60; i++ { + var ( + fields []string + existed bool + ) + output, err := exec.Command("bash", "-c", "kubectl get configuration").Output() + assert.NilError(t, err) + + lines := strings.Split(string(output), "\n") + + for j, line := range lines { + if j == 0 { + continue + } + fields = strings.Fields(line) + if len(fields) == 3 && fields[0] == "alibaba-eip" { + existed = true + } + } + if existed { + if i == 59 { + t.Error("Configuration is not ready") + } + + time.Sleep(time.Second * 5) + continue + } else { + break + } + } + + klog.Info("6. Checking Secrets and ConfigMap which should all be deleted") + _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "variable-alibaba-eip", v1.GetOptions{}) + assert.Equal(t, kerrors.IsNotFound(err), true) + + _, err = clientSet.CoreV1().Secrets("default").Get(ctx, "eip-conn", v1.GetOptions{}) + assert.Equal(t, kerrors.IsNotFound(err), true) + + _, err = clientSet.CoreV1().Secrets(backendSecretNamespace).Get(ctx, "tfstate-default-alibaba-eip", v1.GetOptions{}) + assert.Equal(t, kerrors.IsNotFound(err), true) + + _, err = clientSet.CoreV1().ConfigMaps("default").Get(ctx, "tf-alibaba-eip", v1.GetOptions{}) + assert.Equal(t, kerrors.IsNotFound(err), true) } diff --git a/e2e/configurations/alibaba_test.go b/e2e/configurations/alibaba_test.go index 45d3730e..152f5e46 100644 --- a/e2e/configurations/alibaba_test.go +++ b/e2e/configurations/alibaba_test.go @@ -17,7 +17,6 @@ func TestConfiguration(t *testing.T) { It("All Configurations should become `Available`", func() { pwd, _ := os.Getwd() configurations := []string{ - "examples/alibaba/eip/configuration_eip.yaml", "examples/alibaba/eip/configuration_eip_remote.yaml", "examples/alibaba/eip/configuration_eip_remote_subdirectory.yaml", "examples/alibaba/rds/configuration_hcl_rds.yaml", diff --git a/go.mod b/go.mod index 88ad92a3..7acb3649 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/onsi/ginkgo v1.16.2 github.com/onsi/gomega v1.12.0 github.com/pkg/errors v0.9.1 + golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 google.golang.org/appengine v1.6.5 // indirect gotest.tools v2.2.0+incompatible k8s.io/api v0.18.8