diff --git a/go.mod b/go.mod index 801c40f83..fb9f7bcd1 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.15 // indirect + github.com/klauspost/compress v1.16.3 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect @@ -96,7 +96,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -112,7 +112,7 @@ require ( github.com/wayneashleyberry/terminal-dimensions v1.0.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.7.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sys v0.6.0 // indirect diff --git a/go.sum b/go.sum index a4e5e6c98..cbc8c3c7a 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= +github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -369,8 +369,9 @@ github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/pavlo-v-chernykh/keystore-go/v4 v4.4.1 h1:FyBdsRqqHH4LctMLL+BL2oGO+ONcIPwn96ctofCVtNE= github.com/pavlo-v-chernykh/keystore-go/v4 v4.4.1/go.mod h1:lAVhWwbNaveeJmxrxuSTxMgKpF6DjnuVpn6T8WiBwYQ= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -493,8 +494,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/tests/e2e/const.go b/tests/e2e/const.go index 1b1b86d56..daa85bca9 100644 --- a/tests/e2e/const.go +++ b/tests/e2e/const.go @@ -14,95 +14,97 @@ package e2e -// HelmDescriptors. -var ( - // certManagerHelmDescriptor describes the cert-manager Helm component. - certManagerHelmDescriptor = helmDescriptor{ - Repository: "https://charts.jetstack.io", - ChartName: "cert-manager", - ChartVersion: "v1.11.0", - ReleaseName: "cert-manager", - Namespace: "cert-manager", - SetValues: map[string]string{ - "installCRDs": "true", - }, - RemoteCRDPathVersionTemplate: "https://github.com/jetstack/cert-manager/releases/download/v%s/cert-manager.crds.yaml", - } - - // koperatorLocalHelmDescriptor describes the Koperator Helm component with - // a local chart and version. - koperatorLocalHelmDescriptor = helmDescriptor{ - Repository: "../../charts/kafka-operator", - ChartVersion: LocalVersion, - ReleaseName: "kafka-operator", - Namespace: "kafka", - SetValues: map[string]string{ - "crd.enabled": "true", - }, - LocalCRDSubpaths: []string{"templates/crds.yaml"}, - LocalCRDTemplateRenderValues: map[string]string{ - "crd.enabled": "true", - }, - } - - // koperatorLocalHelmDescriptor describes the Koperator Helm component with - // a remote latest chart and version. - koperatorRemoteLatestHelmDescriptor = helmDescriptor{ //nolint:unused // Note: intentional possibly needed in the future for upgrade test. - Repository: "https://kubernetes-charts.banzaicloud.com", - ChartName: "kafka-operator", - ChartVersion: "", // Note: empty string translates to latest final version. - ReleaseName: "kafka-operator", - Namespace: "kafka", - SetValues: map[string]string{ - "crd.enabled": "true", - }, - RemoteCRDPathVersionTemplate: "https://github.com/banzaicloud/koperator/releases/download/%s/kafka-operator.crds.yaml", - } - - // prometheusOperatorHelmDescriptor describes the prometheus-operator Helm - // component. - prometheusOperatorHelmDescriptor = helmDescriptor{ - Repository: "https://prometheus-community.github.io/helm-charts", - ChartName: "kube-prometheus-stack", - ChartVersion: "42.0.1", - ReleaseName: "prometheus-operator", - Namespace: "prometheus", - SetValues: map[string]string{ - "prometheusOperator.createCustomResource": "true", - "defaultRules.enabled": "false", - "alertmanager.enabled": "false", - "grafana.enabled": "false", - "kubeApiServer.enabled": "false", - "kubelet.enabled": "false", - "kubeControllerManager.enabled": "false", - "coreDNS.enabled": "false", - "kubeEtcd.enabled": "false", - "kubeScheduler.enabled": "false", - "kubeProxy.enabled": "false", - "kubeStateMetrics.enabled": "false", - "nodeExporter.enabled": "false", - "prometheus.enabled": "false", - }, - } - - // zookeeperOperatorHelmDescriptor describes the zookeeper-operator Helm - // component. - zookeeperOperatorHelmDescriptor = helmDescriptor{ - Repository: "https://charts.pravega.io", - ChartName: "zookeeper-operator", - ChartVersion: "0.2.14", - ReleaseName: "zookeeper-operator", - Namespace: "zookeeper", - SetValues: map[string]string{ - "crd.create": "true", - }, - } -) +import "time" // Versions. type Version = string const ( + // LocalVersion means using the files in the local repository snapshot. LocalVersion Version = "local" + + kubectlArgGoTemplateName = `-o=go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'` + kubectlArgGoTemplateKindNameNamespace = `-o=go-template='{{range .items}}{{.kind}}{{"/"}}{{.metadata.name}}{{if .metadata.namespace}}{{"."}}{{.metadata.namespace}}{{end}}{{"\n"}}{{end}}'` + kubectlArgGoTemplateInternalListenersName = `-o=go-template='{{range $key,$value := .status.listenerStatuses.internalListeners}}{{$key}}{{"\n"}}{{end}}` + kubectlArgGoTemplateInternalListenerAddressesTemplate = `-o=go-template='{{range .status.listenerStatuses.internalListeners.%s}}{{.address}}{{"\n"}}{{end}}` + // kubectlArgGoTemplateExternalListenersName = `-o=go-template='{{range $key,$value := .status.listenerStatuses.externallListeners}}{{$key}}{{"\n"}}{{end}}` + // kubectlArgGoTemplateExternalListenerAddressesTemplate = `-o=go-template='{{range .status.listenerStatuses.externalListeners.%s}}{{.address}}{{"\n"}}{{end}}` + + crdKind = "customresourcedefinitions.apiextensions.k8s.io" + kafkaKind = "kafkaclusters.kafka.banzaicloud.io" + kafkaTopicKind = "kafkatopics.kafka.banzaicloud.io" + kafkaClusterName = "kafka" + testTopicName = "topic-test" + kcatPodName = "kcat" + zookeeperKind = "zookeeperclusters.zookeeper.pravega.io" + zookeeperClusterName = "zookeeper-server" + managedByHelmLabelTemplate = "app.kubernetes.io/managed-by=Helm,app.kubernetes.io/instance=%s" + + defaultDeletionTimeout = 20 * time.Second + defaultPodReadinessWaitTime = 10 * time.Second + defaultTopicCreationWaitTime = 10 * time.Second + kafkaClusterResourceCleanupTimeout = 30 * time.Second + zookeeperClusterResourceCleanupTimeout = 60 * time.Second + externalConsumerTimeout = 5 * time.Second + externalProducerTimeout = 5 * time.Second + + kcatPodTemplate = "templates/kcat.yaml.tmpl" + kafkaTopicTemplate = "templates/topic.yaml.tmpl" + + kubectlNotFoundErrorMsg = "NotFound" ) + +func apiGroupKoperatorDependencies() map[string]string { + return map[string]string{ + "cert-manager": "cert-manager.io", + "zookeeper": "zookeeper.pravega.io", + "prometheus": "monitoring.coreos.com", + } +} + +func basicK8sResourceKinds() []string { + return []string{ + "pods", + "services", + "deployments.apps", + "daemonset.apps", + "replicasets.apps", + "statefulsets.apps", + "secrets", + "serviceaccounts", + "configmaps", + "mutatingwebhookconfigurations.admissionregistration.k8s.io", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + "jobs.batch", + "cronjobs.batch", + "poddisruptionbudgets.policy", + "podsecuritypolicies.policy", + "persistentvolumeclaims", + "persistentvolumes", + } +} + +func koperatorCRDs() []string { + return []string{ + "kafkatopics.kafka.banzaicloud.io", + "kafkaclusters.kafka.banzaicloud.io", + "kafkausers.kafka.banzaicloud.io", + "cruisecontroloperations.kafka.banzaicloud.io", + } +} + +func koperatorRelatedResourceKinds() []string { + return []string{ + "nodepoollabelsets.labels.banzaicloud.io", + "kafkatopics.kafka.banzaicloud.io", + "kafkaclusters.kafka.banzaicloud.io", + "kafkausers.kafka.banzaicloud.io", + "cruisecontroloperations.kafka.banzaicloud.io", + "istiomeshgateways.servicemesh.cisco.com", + "virtualservices.networking.istio.io", + "gateways.networking.istio.io", + "clusterissuers.cert-manager.io", + "servicemonitors.monitoring.coreos.com", + } +} diff --git a/tests/e2e/global.go b/tests/e2e/global.go new file mode 100644 index 000000000..ba97b2edb --- /dev/null +++ b/tests/e2e/global.go @@ -0,0 +1,111 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" +) + +// HelmDescriptors. +var ( + // certManagerHelmDescriptor describes the cert-manager Helm component. + certManagerHelmDescriptor = helmDescriptor{ + Repository: "https://charts.jetstack.io", + ChartName: "cert-manager", + ChartVersion: "v1.11.0", + ReleaseName: "cert-manager", + Namespace: "cert-manager", + SetValues: map[string]string{ + "installCRDs": "true", + }, + RemoteCRDPathVersionTemplate: "https://github.com/jetstack/cert-manager/releases/download/v%s/cert-manager.crds.yaml", + } + + // koperatorLocalHelmDescriptor describes the Koperator Helm component with + // a local chart and version. + koperatorLocalHelmDescriptor = helmDescriptor{ + Repository: "../../charts/kafka-operator", + ChartVersion: LocalVersion, + ReleaseName: "kafka-operator", + Namespace: "kafka", + SetValues: map[string]string{ + "crd.enabled": "true", + }, + LocalCRDSubpaths: []string{"templates/crds.yaml"}, + LocalCRDTemplateRenderValues: map[string]string{ + "crd.enabled": "true", + }, + } + + // koperatorLocalHelmDescriptor describes the Koperator Helm component with + // a remote latest chart and version. + koperatorRemoteLatestHelmDescriptor = helmDescriptor{ //nolint:unused // Note: intentional possibly needed in the future for upgrade test. + Repository: "https://kubernetes-charts.banzaicloud.com", + ChartName: "kafka-operator", + ChartVersion: "", // Note: empty string translates to latest final version. + ReleaseName: "kafka-operator", + Namespace: "kafka", + SetValues: map[string]string{ + "crd.enabled": "true", + }, + RemoteCRDPathVersionTemplate: "https://github.com/banzaicloud/koperator/releases/download/%s/kafka-operator.crds.yaml", + } + + // prometheusOperatorHelmDescriptor describes the prometheus-operator Helm + // component. + prometheusOperatorHelmDescriptor = helmDescriptor{ + Repository: "https://prometheus-community.github.io/helm-charts", + ChartName: "kube-prometheus-stack", + ChartVersion: "42.0.1", + ReleaseName: "prometheus-operator", + Namespace: "prometheus", + SetValues: map[string]string{ + "prometheusOperator.createCustomResource": "true", + "defaultRules.enabled": "false", + "alertmanager.enabled": "false", + "grafana.enabled": "false", + "kubeApiServer.enabled": "false", + "kubelet.enabled": "false", + "kubeControllerManager.enabled": "false", + "coreDNS.enabled": "false", + "kubeEtcd.enabled": "false", + "kubeScheduler.enabled": "false", + "kubeProxy.enabled": "false", + "kubeStateMetrics.enabled": "false", + "nodeExporter.enabled": "false", + "prometheus.enabled": "false", + }, + } + + // zookeeperOperatorHelmDescriptor describes the zookeeper-operator Helm + // component. + zookeeperOperatorHelmDescriptor = helmDescriptor{ + Repository: "https://charts.pravega.io", + ChartName: "zookeeper-operator", + ChartVersion: "0.2.14", + ReleaseName: "zookeeper-operator", + Namespace: "zookeeper", + SetValues: map[string]string{ + "crd.create": "true", + }, + } + + // dependencyCRDs storing the Koperator dependencies CRDs name + // It should be initialized once with the Initialize() member function + dependencyCRDs dependencyCRDsType + + // ErrorNotFound is for handling that error case when resource is not found + ErrorNotFound = errors.New("not found") +) diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index 8cb3c9f3d..00dd8e12d 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -14,6 +14,9 @@ require ( ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/aws/aws-sdk-go v1.44.122 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -38,13 +41,16 @@ require ( github.com/gruntwork-io/go-commons v0.8.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index 029a9c038..8db5daf55 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -2,6 +2,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT emperror.dev/errors v0.8.1 h1:UavXZ5cSX/4u9iyvH6aDcuGkVjeexUGJ7Ij7G4VfQT0= emperror.dev/errors v0.8.1/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= @@ -90,6 +96,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -119,8 +127,12 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/tests/e2e/helm.go b/tests/e2e/helm.go index 8a39173e0..b9814fd0f 100644 --- a/tests/e2e/helm.go +++ b/tests/e2e/helm.go @@ -74,7 +74,7 @@ func (helmDescriptor *helmDescriptor) crdPath() (string, error) { //nolint:unuse // installHelmChart checks whether the specified named Helm release exists in // the provided kubectl context and namespace, logs it if it does and returns or // alternatively deploys a Helm chart to the specified kubectl context and -// namespace using the specified infos, extra arguments can be any of the helm +// namespace using the specified info, extra arguments can be any of the helm // CLI install flag arguments, flag keys and values must be provided separately. func (helmDescriptor *helmDescriptor) installHelmChart(kubectlOptions k8s.KubectlOptions) error { if helmDescriptor == nil { @@ -193,6 +193,64 @@ func (helmDescriptor *helmDescriptor) installHelmChart(kubectlOptions k8s.Kubect return nil } +// uninstallHelmChart checks whether the specified named Helm release exists in +// the provided kubectl context and namespace, logs it if it does not and when noErrorNotFound is false then it returns error. +// if the Helm chart present then it uninstalls it from the specified kubectl context +// and namespace using the specified info, extra arguments can be any of the helm +// CLI install flag arguments, flag keys and values must be provided separately. +func (helmDescriptor *helmDescriptor) uninstallHelmChart(kubectlOptions k8s.KubectlOptions, noErrorNotFound bool) error { + if helmDescriptor == nil { + return errors.Errorf("invalid nil Helm descriptor") + } + + kubectlOptions.Namespace = helmDescriptor.Namespace + + By(fmt.Sprintf("Checking for existing Helm release named %s", helmDescriptor.ReleaseName)) + _, isInstalled, err := lookUpInstalledHelmReleaseByName(kubectlOptions, helmDescriptor.ReleaseName) + if err != nil { + return errors.WrapIfWithDetails( + err, + "looking up Helm release failed", + "releaseName", helmDescriptor.ReleaseName, + ) + } + + if !isInstalled { + if !noErrorNotFound { + return errors.Errorf("Helm release: '%s' not found", helmDescriptor.ReleaseName) + } + + By(fmt.Sprintf( + "skipping the uninstallation of %s, because the Helm release is not present.", + helmDescriptor.ReleaseName, + )) + return nil + } + By( + fmt.Sprintf( + "uninstalling Helm chart by name %s", + helmDescriptor.ReleaseName, + ), + ) + + fixedArguments := []string{ + "--debug", + } + purge := true + + return helm.DeleteE( + GinkgoT(), + &helm.Options{ + KubectlOptions: &kubectlOptions, + ExtraArgs: map[string][]string{ + "delete": append(fixedArguments, helmDescriptor.HelmExtraArguments["install"]...), + }, + }, + helmDescriptor.ReleaseName, + purge, + ) +} + // IsRemote returns true when the Helm descriptor uses a remote chart path as // location. In any other case the repository is considered a remote Helm // repository URL. diff --git a/tests/e2e/k8s.go b/tests/e2e/k8s.go index 2b2f807a9..69298a644 100644 --- a/tests/e2e/k8s.go +++ b/tests/e2e/k8s.go @@ -22,9 +22,11 @@ import ( "os" "path" "strings" + "text/template" "time" "emperror.dev/errors" + "github.com/Masterminds/sprig" "github.com/cisco-open/k8s-objectmatcher/patch" "github.com/gruntwork-io/terratest/modules/k8s" . "github.com/onsi/ginkgo/v2" @@ -392,3 +394,189 @@ func listK8sCRDs(kubectlOptions k8s.KubectlOptions, crdNames ...string) ([]strin return strings.Split(output, "\n"), nil } + +// deleteK8sResourceOpts deletes K8s resources based on the kind and name or kind and selector. +// It returns error in case when the resource is not found. +// timeout parameter specifies the timeout for the deletion. +// extraArgs can be any of the kubectl arguments. +func deleteK8sResource( + kubectlOptions k8s.KubectlOptions, + timeout time.Duration, + kind string, + selector string, + name string, + extraArgs ...string) error { + + args := []string{"delete", kind} + + args = append(args, fmt.Sprintf("--timeout=%s", timeout)) + + logMsg := fmt.Sprintf("Deleting k8s resource: kind: '%s' ", kind) + logMsg, args = kubectlArgExtender(args, logMsg, selector, name, kubectlOptions.Namespace, extraArgs) + By(logMsg) + + _, err := k8s.RunKubectlAndGetOutputE( + GinkgoT(), + &kubectlOptions, + args..., + ) + + return err +} + +// deleteK8sResourceNoErrNotFound deletes K8s resources based on the kind and name or kind and selector. +// Deletion is passed in case the resource is not found. +// timeout parameter specifies the timeout for the deletion. +// extraArgs can be any of the kubectl arguments. +func deleteK8sResourceNoErrNotFound(kubectlOptions k8s.KubectlOptions, timeout time.Duration, kind string, name string, extraArgs ...string) error { + err := deleteK8sResource(kubectlOptions, timeout, kind, "", name, extraArgs...) + if isKubectlNotFoundError(err) { + By(fmt.Sprintf("K8s resource %s not found", name)) + return nil + } + return err +} + +// applyK8sResourceManifestFromString applies the specified manifest in string format to the provided +// kubectl context and namespace. +func applyK8sResourceManifestFromString(kubectlOptions k8s.KubectlOptions, manifest string) error { + By(fmt.Sprintf("Applying k8s manifest\n%s", manifest)) + return k8s.KubectlApplyFromStringE(GinkgoT(), &kubectlOptions, manifest) +} + +// applyK8sResourceFromTemplate generates manifest from the specified go-template based on values +// and applies the specified manifest to the provided kubectl context and namespace. +func applyK8sResourceFromTemplate(kubectlOptions k8s.KubectlOptions, templateFile string, values map[string]interface{}) error { + By(fmt.Sprintf("Generating K8s manifest from template %s", templateFile)) + var manifest bytes.Buffer + rawTemplate, err := os.ReadFile(templateFile) + if err != nil { + return err + } + t := template.Must(template.New("template").Funcs(sprig.TxtFuncMap()).Parse(string(rawTemplate))) + err = t.Execute(&manifest, values) + if err != nil { + return err + } + return applyK8sResourceManifestFromString(kubectlOptions, manifest.String()) +} + +// listK8sResourceKinds lists all of the available resource kinds on the K8s cluster +// with the apiGroupSelector parameter the result can be narrowed by the resource group. +// extraArgs can be any kubectl api-resources parameter. +func listK8sResourceKinds(kubectlOptions k8s.KubectlOptions, apiGroupSelector string, extraArgs ...string) ([]string, error) { + logMsg := "Listing K8s resource kind" + args := []string{"api-resources", "--verbs", "list", "--output", "name", "--sort-by", "name"} + + if apiGroupSelector != "" { + logMsg = fmt.Sprintf("%s group selector: %s", logMsg, apiGroupSelector) + args = append(args, "--api-group", apiGroupSelector) + } + + args = append(args, extraArgs...) + + output, err := k8s.RunKubectlAndGetOutputE( + GinkgoT(), + &kubectlOptions, + args..., + ) + + if err != nil { + return nil, err + } + + return kubectlRemoveWarnings(strings.Split(output, "\n")), nil +} + +// getK8sResources gets the specified K8S resources from the specified kubectl context and +// namespace optionally. Extra arguments can be any of the kubectl get flag arguments. +// Returns a slice of the returned elements. Separator between elements must be newline. +func getK8sResources(kubectlOptions k8s.KubectlOptions, resourceKind []string, selector string, names string, extraArgs ...string) ([]string, error) { + logMsg := fmt.Sprintf("Get K8S resources: '%s'", resourceKind) + + args := []string{"get", strings.Join(resourceKind, ",")} + logMsg, args = kubectlArgExtender(args, logMsg, selector, names, kubectlOptions.Namespace, extraArgs) + By(logMsg) + + output, err := k8s.RunKubectlAndGetOutputE( + GinkgoT(), + &kubectlOptions, + args..., + ) + + if err != nil { + return nil, err + } + + output = strings.Trim(output, "'") + // Empty output + if output == "" { + return nil, nil + } + + output = strings.TrimRight(output, "\n") + outputSlice := strings.Split(output, "\n") + + // Remove warning message pollution from the output + + return kubectlRemoveWarnings(outputSlice), nil +} + +// waitK8sResourceCondition waits until the condition is met or the timeout is elapsed for the selected K8s resource(s) +// extraArgs can be any of the kubectl arguments +func waitK8sResourceCondition(kubectlOptions k8s.KubectlOptions, resourceKind, waitFor string, timeout time.Duration, selector string, names string, extraArgs ...string) error { + logMsg := fmt.Sprintf("Waiting K8s resource(s)' condition: '%s' to fulfil", waitFor) + + args := []string{ + "wait", + resourceKind, + fmt.Sprintf("--for=%s", waitFor), + fmt.Sprintf("--timeout=%s", timeout), + } + + logMsg, args = kubectlArgExtender(args, logMsg, selector, names, kubectlOptions.Namespace, extraArgs) + By(logMsg) + + _, err := k8s.RunKubectlAndGetOutputE( + GinkgoT(), + &kubectlOptions, + args..., + ) + + return err +} + +// kubectlArgExtender extends the kubectl arguments and log message based on the parameters +func kubectlArgExtender(args []string, logMsg, selector, names, namespace string, extraArgs []string) (string, []string) { + if selector != "" { + logMsg = fmt.Sprintf("%s selector: '%s'", logMsg, selector) + args = append(args, fmt.Sprintf("--selector=%s", selector)) + } else if names != "" { + logMsg = fmt.Sprintf("%s name(s): '%s'", logMsg, names) + args = append(args, names) + } + if namespace != "" { + logMsg = fmt.Sprintf("%s namespace: '%s'", logMsg, namespace) + } + if len(extraArgs) != 0 { + logMsg = fmt.Sprintf("%s extraArgs: '%s'", logMsg, extraArgs) + args = append(args, extraArgs...) + } + return logMsg, args +} + +// kubectlRemoveWarning removes those elements from the outputSlice parameter which contains kubectl warning message. +func kubectlRemoveWarnings(outputSlice []string) []string { + // Remove warning message pollution from the output + result := make([]string, 0, len(outputSlice)) + for i := range outputSlice { + if !strings.Contains(outputSlice[i], "Warning:") { + result = append(result, outputSlice[i]) + } + } + return result +} + +func isKubectlNotFoundError(err error) bool { + return err != nil && strings.Contains(err.Error(), kubectlNotFoundErrorMsg) +} diff --git a/tests/e2e/kafka.go b/tests/e2e/kafka.go new file mode 100644 index 000000000..7553913fe --- /dev/null +++ b/tests/e2e/kafka.go @@ -0,0 +1,50 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// requireDeleteKafkaTopic deletes kafkaTopic resource. +func requireDeleteKafkaTopic(kubectlOptions k8s.KubectlOptions, topicName string) { + It("Deleting KafkaTopic CR", func() { + err := deleteK8sResource(kubectlOptions, defaultDeletionTimeout, kafkaTopicKind, "", topicName) + Expect(err).NotTo(HaveOccurred()) + }) +} + +// requireDeployingKafkaTopic deploys a kafkaTopic resource from a template +func requireDeployingKafkaTopic(kubectlOptions k8s.KubectlOptions, topicName string) { + It("Deploying KafkaTopic CR", func() { + err := applyK8sResourceFromTemplate(kubectlOptions, + kafkaTopicTemplate, + map[string]interface{}{ + "Name": topicName, + "TopicName": topicName, + "Namespace": kubectlOptions.Namespace, + }, + ) + Expect(err).ShouldNot(HaveOccurred()) + + err = waitK8sResourceCondition(kubectlOptions, kafkaTopicKind, + "jsonpath={.status.state}=created", defaultTopicCreationWaitTime, "", topicName) + + Expect(err).ShouldNot(HaveOccurred()) + }) + +} diff --git a/tests/e2e/kcat.go b/tests/e2e/kcat.go new file mode 100644 index 000000000..784a723b2 --- /dev/null +++ b/tests/e2e/kcat.go @@ -0,0 +1,59 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" +) + +// consumingMessagesInternally consuming messages based on parameters from Kafka cluster. +// It returns messages in string slice. +func consumingMessagesInternally(kubectlOptions k8s.KubectlOptions, kcatPodName string, internalKafkaAddress string, topicName string) (string, error) { + + By(fmt.Sprintf("Consuming messages from internalKafkaAddress: '%s' topicName: '%s'", internalKafkaAddress, topicName)) + + consumedMessages, err := k8s.RunKubectlAndGetOutputE(GinkgoT(), + k8s.NewKubectlOptions(kubectlOptions.ContextName, kubectlOptions.ConfigPath, ""), + "exec", kcatPodName, + "-n", kubectlOptions.Namespace, + "--", + "/bin/sh", "-c", fmt.Sprintf("kcat -L -b %s -t %s -e -C ", internalKafkaAddress, topicName), + ) + + if err != nil { + return "", err + } + + return consumedMessages, nil +} + +// producingMessagesInternally produces messages based on the parameters into kafka cluster. +func producingMessagesInternally(kubectlOptions k8s.KubectlOptions, kcatPodName string, internalKafkaAddress string, topicName string, message string) error { + By(fmt.Sprintf("Producing messages: '%s' to internalKafkaAddress: '%s' topicName: '%s'", message, internalKafkaAddress, topicName)) + + _, err := k8s.RunKubectlAndGetOutputE(GinkgoT(), + k8s.NewKubectlOptions(kubectlOptions.ContextName, kubectlOptions.ConfigPath, ""), + "exec", kcatPodName, + "-n", kubectlOptions.Namespace, + "--", + "/bin/sh", "-c", fmt.Sprintf("echo %s | kcat -L -b %s -t %s -P", + message, internalKafkaAddress, topicName), + ) + + return err +} diff --git a/tests/e2e/koperator_suite_test.go b/tests/e2e/koperator_suite_test.go index 5be005541..5527a13b5 100644 --- a/tests/e2e/koperator_suite_test.go +++ b/tests/e2e/koperator_suite_test.go @@ -52,3 +52,11 @@ var _ = BeforeSuite(func() { Expect(len(pods)).To(Not(BeZero())) }) }) + +var _ = When("Testing e2e test altogether", Ordered, func() { + //testInstall() + //testProduceConsumeInternal() + testUninstallZookeeperCluster() + testUninstallKafkaCluster() + testUninstall() +}) diff --git a/tests/e2e/produce_consume.go b/tests/e2e/produce_consume.go new file mode 100644 index 000000000..f0777c01b --- /dev/null +++ b/tests/e2e/produce_consume.go @@ -0,0 +1,93 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "time" + + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// requireDeployingKcatPod deploys kcat pod form a template and checks the pod readiness +func requireDeployingKcatPod(kubectlOptions k8s.KubectlOptions, podName string) { + It("Deploying Kcat Pod", func() { + err := applyK8sResourceFromTemplate(kubectlOptions, + kcatPodTemplate, + map[string]interface{}{ + "Name": kcatPodName, + "Namespace": kubectlOptions.Namespace, + }, + ) + Expect(err).ShouldNot(HaveOccurred()) + + err = waitK8sResourceCondition(kubectlOptions, "pods", + "condition=Ready", defaultPodReadinessWaitTime, "", kcatPodName) + + Expect(err).ShouldNot(HaveOccurred()) + }) + +} + +// requireDeleteKcatPod deletes kcat pod. +func requireDeleteKcatPod(kubectlOptions k8s.KubectlOptions, podName string) { + It("Deleting Kcat pod", func() { + err := deleteK8sResource(kubectlOptions, defaultDeletionTimeout, "pods", "", podName) + Expect(err).NotTo(HaveOccurred()) + }) +} + +// requireInternalProducingConsumingMessage produces and consumes messages internally through a kcat pod +// and makes comparisons between the produced and consumed messages. +// When internalAddress parameter is empty, it gets the internal address from the kafkaCluster CR status +func requireInternalProducingConsumingMessage(kubectlOptions k8s.KubectlOptions, internalAddress, kcatPodName, topicName string) { + It(fmt.Sprintf("Producing and consuming messages to/from topicName: '%s", topicName), func() { + if internalAddress == "" { + By("Getting Kafka cluster internal addresses") + internalListenerNames, err := getK8sResources(kubectlOptions, + []string{kafkaKind}, + "", + kafkaClusterName, + kubectlArgGoTemplateInternalListenersName, + ) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(internalListenerNames).ShouldNot(BeEmpty()) + + internalListenerAddresses, err := getK8sResources(kubectlOptions, + []string{kafkaKind}, + "", + kafkaClusterName, + fmt.Sprintf(kubectlArgGoTemplateInternalListenerAddressesTemplate, internalListenerNames[0]), + ) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(internalListenerAddresses).ShouldNot(BeEmpty()) + + internalAddress = internalListenerAddresses[0] + } + + currentTime := time.Now() + err := producingMessagesInternally(kubectlOptions, kcatPodName, internalAddress, topicName, currentTime.String()) + Expect(err).NotTo(HaveOccurred()) + + consumedMessages, err := consumingMessagesInternally(kubectlOptions, kcatPodName, internalAddress, topicName) + + Expect(err).NotTo(HaveOccurred()) + Expect(consumedMessages).Should(ContainSubstring(currentTime.String())) + }) +} diff --git a/tests/e2e/templates/kcat.yaml.tmpl b/tests/e2e/templates/kcat.yaml.tmpl new file mode 100644 index 000000000..47e783070 --- /dev/null +++ b/tests/e2e/templates/kcat.yaml.tmpl @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{or .Name "kcat"}} + namespace: {{or .Namespace "kafka" }} +spec: + serviceAccount: {{or .ServiceAccount "default"}} + containers: + - name: kafka-client + image: edenhill/kcat:1.7.0 + # Just spin & wait forever + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 3000; done;" ] diff --git a/tests/e2e/templates/topic.yaml.tmpl b/tests/e2e/templates/topic.yaml.tmpl new file mode 100644 index 000000000..991baffe8 --- /dev/null +++ b/tests/e2e/templates/topic.yaml.tmpl @@ -0,0 +1,15 @@ +apiVersion: kafka.banzaicloud.io/v1alpha1 +kind: KafkaTopic +metadata: + name: {{ .Name }} + namespace: {{or .Namespace "kafka"}} +spec: + clusterRef: + name: kafka + namespace: kafka + name: {{ .TopicName }} + partitions: {{or .Partition 2}} + replicationFactor: {{or .ReplicationFactor 2}} + config: + "retention.ms": "604800000" + "cleanup.policy": "delete" diff --git a/tests/e2e/test_install.go b/tests/e2e/test_install.go index d19f5d3cb..185065644 100644 --- a/tests/e2e/test_install.go +++ b/tests/e2e/test_install.go @@ -20,10 +20,8 @@ import ( . "github.com/onsi/gomega" ) -var _ = testInstall() - func testInstall() bool { - return When("Installing Koperator", Ordered, func() { + return When("Installing Koperator and dependencies", Ordered, func() { var kubectlOptions k8s.KubectlOptions var err error diff --git a/tests/e2e/test_produce_consume.go b/tests/e2e/test_produce_consume.go new file mode 100644 index 000000000..b2ba7c73e --- /dev/null +++ b/tests/e2e/test_produce_consume.go @@ -0,0 +1,27 @@ +package e2e + +import ( + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func testProduceConsumeInternal() bool { + return When("Internally produce and consume message to/from Kafka cluster", func() { + var kubectlOptions k8s.KubectlOptions + var err error + + It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + Expect(err).NotTo(HaveOccurred()) + }) + + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + + requireDeployingKcatPod(kubectlOptions, kcatPodName) + requireDeployingKafkaTopic(kubectlOptions, testTopicName) + requireInternalProducingConsumingMessage(kubectlOptions, "", kcatPodName, testTopicName) + requireDeleteKafkaTopic(kubectlOptions, testTopicName) + requireDeleteKcatPod(kubectlOptions, kcatPodName) + }) +} diff --git a/tests/e2e/test_uninstall.go b/tests/e2e/test_uninstall.go new file mode 100644 index 000000000..6c1bdaf3d --- /dev/null +++ b/tests/e2e/test_uninstall.go @@ -0,0 +1,63 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func testUninstall() bool { + return When("Uninstalling Koperator and dependencies", Ordered, func() { + var kubectlOptions k8s.KubectlOptions + var err error + + When("Initializing", func() { + It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + Expect(err).NotTo(HaveOccurred()) + }) + + It("Setting globals", func() { + err := dependencyCRDs.Initialize(kubectlOptions) + Expect(err).NotTo(HaveOccurred()) + }) + + }) + + requireUninstallingKoperator(k8s.KubectlOptions{ + ContextName: kubectlOptions.ContextName, + ConfigPath: kubectlOptions.ConfigPath, + Namespace: koperatorLocalHelmDescriptor.Namespace, + }) + requireUninstallingZookeeperOperator(k8s.KubectlOptions{ + ContextName: kubectlOptions.ContextName, + ConfigPath: kubectlOptions.ConfigPath, + Namespace: zookeeperOperatorHelmDescriptor.Namespace, + }) + requireUninstallingPrometheusOperator(k8s.KubectlOptions{ + ContextName: kubectlOptions.ContextName, + ConfigPath: kubectlOptions.ConfigPath, + Namespace: prometheusOperatorHelmDescriptor.Namespace, + }) + requireUninstallingCertManager(k8s.KubectlOptions{ + ContextName: kubectlOptions.ContextName, + ConfigPath: kubectlOptions.ConfigPath, + Namespace: certManagerHelmDescriptor.Namespace, + }) + + }) +} diff --git a/tests/e2e/test_uninstall_cluster.go b/tests/e2e/test_uninstall_cluster.go new file mode 100644 index 000000000..78d71aad1 --- /dev/null +++ b/tests/e2e/test_uninstall_cluster.go @@ -0,0 +1,52 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func testUninstallZookeeperCluster() bool { + return When("Uninstalling Zookeeper cluster", func() { + var kubectlOptions k8s.KubectlOptions + var err error + + It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + Expect(err).NotTo(HaveOccurred()) + }) + + kubectlOptions.Namespace = zookeeperOperatorHelmDescriptor.Namespace + requireDeleteZookeeperCluster(kubectlOptions, zookeeperClusterName) + + }) +} + +func testUninstallKafkaCluster() bool { + return When("Uninstalling Kafka cluster", func() { + var kubectlOptions k8s.KubectlOptions + var err error + + It("Acquiring K8s config and context", func() { + kubectlOptions, err = kubectlOptionsForCurrentContext() + Expect(err).NotTo(HaveOccurred()) + }) + + kubectlOptions.Namespace = koperatorLocalHelmDescriptor.Namespace + requireDeleteKafkaCluster(kubectlOptions, kafkaClusterName) + }) +} diff --git a/tests/e2e/types.go b/tests/e2e/types.go new file mode 100644 index 000000000..c53b9776d --- /dev/null +++ b/tests/e2e/types.go @@ -0,0 +1,49 @@ +package e2e + +import ( + "fmt" + + "github.com/gruntwork-io/terratest/modules/k8s" +) + +type dependencyCRDsType struct { + zookeeper []string + prometheus []string + certManager []string +} + +func (c *dependencyCRDsType) Zookeeper() []string { + return c.zookeeper +} +func (c *dependencyCRDsType) Prometheus() []string { + return c.prometheus +} +func (c *dependencyCRDsType) CertManager() []string { + return c.certManager +} + +func (c *dependencyCRDsType) Initialize(kubectlOptions k8s.KubectlOptions) error { + var err error + c.certManager, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["cert-manager"]) + if len(c.certManager) == 0 { + if err != nil { + return fmt.Errorf("initialize Cert-manager CRDs error: %w", err) + } + return fmt.Errorf("Cert-manager CRDs %w", ErrorNotFound) + } + c.prometheus, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["prometheus"]) + if len(c.prometheus) == 0 { + if err != nil { + return fmt.Errorf("initialize Prometheus CRDs error: %w", err) + } + return fmt.Errorf("Prometheus CRDs %w", ErrorNotFound) + } + c.zookeeper, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["zookeeper"]) + if len(c.zookeeper) == 0 { + if err != nil { + return fmt.Errorf("initialize Zookeeper CRDs error: %w", err) + } + return fmt.Errorf("Zookeeper CRDs %w", ErrorNotFound) + } + return nil +} diff --git a/tests/e2e/uninstall.go b/tests/e2e/uninstall.go new file mode 100644 index 000000000..4644fb064 --- /dev/null +++ b/tests/e2e/uninstall.go @@ -0,0 +1,207 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// requireRemoveKoperatorCRDs deletes the koperator CRDs +func requireRemoveKoperatorCRDs(kubectlOptions k8s.KubectlOptions) { + It("Removing koperator CRDs", func() { + for _, crd := range koperatorCRDs() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, crdKind, crd) + Expect(err).ShouldNot(HaveOccurred()) + + } + }) +} + +// requireUninstallingKoperator uninstall koperator Helm chart and removes Koperator's CRDs. +func requireUninstallingKoperator(kubectlOptions k8s.KubectlOptions) { + When("Uninstalling Koperator", func() { + requireUninstallingKoperatorHelmChart(kubectlOptions) + requireRemoveKoperatorCRDs(kubectlOptions) + }) +} + +// requireUninstallingKoperatorHelmChart uninstall Koperator Helm chart +// and checks the success of that operation. +func requireUninstallingKoperatorHelmChart(kubectlOptions k8s.KubectlOptions) { + It("Uninstalling Koperator Helm chart", func() { + err := koperatorLocalHelmDescriptor.uninstallHelmChart(kubectlOptions, true) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Koperator helm chart resources cleanup") + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + Expect(err).ShouldNot(HaveOccurred()) + + koperatorAvailableResourceKinds := stringSlicesInstersect(koperatorCRDs(), k8sResourceKinds) + koperatorAvailableResourceKinds = append(koperatorAvailableResourceKinds, basicK8sResourceKinds()...) + + remainedResources, err := getK8sResources(kubectlOptions, + koperatorAvailableResourceKinds, + fmt.Sprintf(managedByHelmLabelTemplate, koperatorLocalHelmDescriptor.ReleaseName), + "", + kubectlArgGoTemplateKindNameNamespace, + "--all-namespaces") + + Expect(err).ShouldNot(HaveOccurred()) + Expect(remainedResources).Should(BeEmpty()) + }) +} + +// requireUninstallingZookeeperOperator uninstall Zookeeper-operator Helm chart +// and remove CRDs. +func requireUninstallingZookeeperOperator(kubectlOptions k8s.KubectlOptions) { + When("Uninstalling zookeeper-operator", func() { + requireUninstallingZookeeperOperatorHelmChart(kubectlOptions) + requireRemoveZookeeperOperatorCRDs(kubectlOptions) + }) +} + +// requireUninstallingZookeeperOperatorHelmChart uninstall Zookeeper-operator Helm chart +// and checks the success of that operation. +func requireUninstallingZookeeperOperatorHelmChart(kubectlOptions k8s.KubectlOptions) { + It("Uninstalling zookeeper-operator Helm chart", func() { + err := zookeeperOperatorHelmDescriptor.uninstallHelmChart(kubectlOptions, true) + Expect(err).NotTo(HaveOccurred()) + By("Verifying Zookeeper-operator helm chart resources cleanup") + + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + Expect(err).ShouldNot(HaveOccurred()) + + zookeeperAvailableResourceKinds := stringSlicesInstersect(dependencyCRDs.Zookeeper(), k8sResourceKinds) + zookeeperAvailableResourceKinds = append(zookeeperAvailableResourceKinds, basicK8sResourceKinds()...) + + remainedResources, err := getK8sResources(kubectlOptions, + zookeeperAvailableResourceKinds, + fmt.Sprintf(managedByHelmLabelTemplate, zookeeperOperatorHelmDescriptor.ReleaseName), + "", + kubectlArgGoTemplateKindNameNamespace, + "--all-namespaces") + Expect(err).ShouldNot(HaveOccurred()) + + Expect(remainedResources).Should(BeEmpty()) + }) +} + +// requireRemoveZookeeperOperatorCRDs deletes the zookeeper-operator CRDs +func requireRemoveZookeeperOperatorCRDs(kubectlOptions k8s.KubectlOptions) { + It("Removing zookeeper-operator CRDs", func() { + for _, crd := range dependencyCRDs.Zookeeper() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, crdKind, crd) + Expect(err).ShouldNot(HaveOccurred()) + } + }) +} + +// requireUninstallingPrometheusOperator uninstall prometheus-operator Helm chart and +// remove CRDs. +func requireUninstallingPrometheusOperator(kubectlOptions k8s.KubectlOptions) { + When("Uninstalling prometheus-operator", func() { + requireUninstallingPrometheusOperatorHelmChart(kubectlOptions) + requireRemovePrometheusOperatorCRDs(kubectlOptions) + }) +} + +// requireUninstallingPrometheusOperatorHelmChart uninstall prometheus-operator Helm chart +// and checks the success of that operation. +func requireUninstallingPrometheusOperatorHelmChart(kubectlOptions k8s.KubectlOptions) { + It("Uninstalling Prometheus-operator Helm chart", func() { + err := prometheusOperatorHelmDescriptor.uninstallHelmChart(kubectlOptions, true) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Prometheus-operator helm chart resources cleanup") + + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + Expect(err).ShouldNot(HaveOccurred()) + + prometheusAvailableResourceKinds := stringSlicesInstersect(dependencyCRDs.Prometheus(), k8sResourceKinds) + prometheusAvailableResourceKinds = append(prometheusAvailableResourceKinds, basicK8sResourceKinds()...) + + remainedResources, err := getK8sResources(kubectlOptions, + prometheusAvailableResourceKinds, + fmt.Sprintf(managedByHelmLabelTemplate, prometheusOperatorHelmDescriptor.ReleaseName), + "", + kubectlArgGoTemplateKindNameNamespace, + "--all-namespaces") + Expect(err).ShouldNot(HaveOccurred()) + + Expect(remainedResources).Should(BeEmpty()) + }) +} + +// requireRemovePrometheusOperatorCRDs deletes the Prometheus-operator CRDs +func requireRemovePrometheusOperatorCRDs(kubectlOptions k8s.KubectlOptions) { + It("Removing prometheus-operator CRDs", func() { + for _, crd := range dependencyCRDs.Prometheus() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, crdKind, crd) + Expect(err).ShouldNot(HaveOccurred()) + } + }) +} + +// requireUninstallingCertManager uninstall Cert-manager Helm chart and +// remove CRDs. +func requireUninstallingCertManager(kubectlOptions k8s.KubectlOptions) { + When("Uninstalling zookeeper-operator", func() { + requireUninstallingCertManagerHelmChart(kubectlOptions) + requireRemoveCertManagerCRDs(kubectlOptions) + }) +} + +// requireUninstallingCertManagerHelmChart uninstalls cert-manager helm chart +// and checks the success of that operation. +func requireUninstallingCertManagerHelmChart(kubectlOptions k8s.KubectlOptions) { + It("Uninstalling Cert-manager Helm chart", func() { + + err := certManagerHelmDescriptor.uninstallHelmChart(kubectlOptions, true) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Cert-manager helm chart resources cleanup") + + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + Expect(err).ShouldNot(HaveOccurred()) + + certManagerAvailableResourceKinds := stringSlicesInstersect(dependencyCRDs.CertManager(), k8sResourceKinds) + certManagerAvailableResourceKinds = append(certManagerAvailableResourceKinds, basicK8sResourceKinds()...) + + remainedResources, err := getK8sResources(kubectlOptions, + certManagerAvailableResourceKinds, + fmt.Sprintf(managedByHelmLabelTemplate, certManagerHelmDescriptor.ReleaseName), + "", + kubectlArgGoTemplateKindNameNamespace, + "--all-namespaces") + Expect(err).ShouldNot(HaveOccurred()) + + Expect(remainedResources).Should(BeEmpty()) + }) + +} + +// requireRemoveKoperatorCRDs deletes the cert-manager CRDs +func requireRemoveCertManagerCRDs(kubectlOptions k8s.KubectlOptions) { + It("Removing cert-manager CRDs", func() { + for _, crd := range dependencyCRDs.CertManager() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, crdKind, crd) + Expect(err).ShouldNot(HaveOccurred()) + } + }) +} diff --git a/tests/e2e/uninstall_cluster.go b/tests/e2e/uninstall_cluster.go new file mode 100644 index 000000000..585224a77 --- /dev/null +++ b/tests/e2e/uninstall_cluster.go @@ -0,0 +1,79 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "time" + + koperator_v1beta1 "github.com/banzaicloud/koperator/api/v1beta1" + + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// requireDeleteKafkaCluster deletes KafkaCluster resource and +// checks the removal of the Kafka cluster related resources +func requireDeleteKafkaCluster(kubectlOptions k8s.KubectlOptions, name string) { + It("Delete KafkaCluster custom resource", func() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, kafkaKind, koperatorLocalHelmDescriptor.Namespace) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(context.Background(), func() []string { + By("Verifying the Kafka cluster resource cleanup") + + // Check only those Koperator related resource types we have in K8s (istio usecase) + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + Expect(err).ShouldNot(HaveOccurred()) + + koperatorAvailableResourceKinds := stringSlicesInstersect(koperatorRelatedResourceKinds(), k8sResourceKinds) + koperatorAvailableResourceKinds = append(koperatorAvailableResourceKinds, basicK8sResourceKinds()...) + + resources, err := getK8sResources(kubectlOptions, + koperatorAvailableResourceKinds, + fmt.Sprintf("%s=%s", koperator_v1beta1.KafkaCRLabelKey, name), + "", + "--all-namespaces", kubectlArgGoTemplateKindNameNamespace) + Expect(err).ShouldNot(HaveOccurred()) + + return resources + }, kafkaClusterResourceCleanupTimeout, 1*time.Second).Should(BeEmpty()) + }) +} + +// requireDeleteZookeeperCluster deletes the ZookeeperCluster CR and verify the corresponding resources cleanup +func requireDeleteZookeeperCluster(kubectlOptions k8s.KubectlOptions, name string) { + It("Delete ZookeeperCluster custom resource", func() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, zookeeperKind, name) + Expect(err).ShouldNot(HaveOccurred()) + + Eventually(context.Background(), func() []string { + By("Verifying the Zookeeper cluster resource cleanup") + + zookeeperK8sResources := basicK8sResourceKinds() + zookeeperK8sResources = append(zookeeperK8sResources, dependencyCRDs.Zookeeper()...) + + resources, err := getK8sResources(kubectlOptions, + zookeeperK8sResources, + fmt.Sprintf("app=%s", name), + "", + "--all-namespaces", kubectlArgGoTemplateKindNameNamespace) + Expect(err).ShouldNot(HaveOccurred()) + return resources + + }, zookeeperClusterResourceCleanupTimeout, 1*time.Second).Should(BeEmpty()) + }) +} diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go new file mode 100644 index 000000000..855524d2c --- /dev/null +++ b/tests/e2e/utils.go @@ -0,0 +1,45 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "sort" +) + +// stringSlicesInstersect returns the union of the slices from argument. +func stringSlicesInstersect(sliceA, sliceB []string) []string { + if len(sliceA) == 0 || len(sliceB) == 0 { + return nil + } + + sort.Slice(sliceA, func(i int, j int) bool { + return sliceA[i] < sliceA[j] + }) + sort.Slice(sliceB, func(i int, j int) bool { + return sliceB[i] < sliceB[j] + }) + + var union []string + for i := range sliceB { + for j := range sliceA { + if sliceB[i] < sliceA[j] { + break + } else if sliceB[i] == sliceA[j] { + union = append(union, sliceB[i]) + } + } + } + return union +}