From ffa6adb44410eba9af890caf582eb3bdceb184de Mon Sep 17 00:00:00 2001 From: Caleb Warren Date: Thu, 14 Mar 2024 09:02:02 -0700 Subject: [PATCH] Add PSACT test files for Qase coverage Merge pull request #44597 from markusewalker/markusv29/task-1173 [v2.9] Add PSACT test files for Qase coverage add vsphere chart validation to automation framework --- go.mod | 2 +- go.sum | 4 +- .../catalogv2/rancher_managed_charts_test.go | 4 +- .../v2/validation/provisioning/k3s/README.md | 3 +- .../provisioning/permutations/permutations.go | 280 ++++++++++++++---- .../v2/validation/provisioning/rke1/README.md | 14 + .../rke1/provisioning_cloud_provider_test.go | 37 +++ .../v2/validation/provisioning/rke2/README.md | 11 +- .../rke2/provisioning_cloud_provider_test.go | 38 +++ 9 files changed, 332 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index 931c08d9961..eb41461a55c 100644 --- a/go.mod +++ b/go.mod @@ -166,7 +166,7 @@ require ( github.com/containers/image/v5 v5.25.0 github.com/google/gnostic-models v0.6.8 github.com/rancher/rancher/pkg/apis v0.0.0-20240213233515-935d309ebad4 - github.com/rancher/shepherd v0.0.0-20240408151625-d0c3b8dbe5dd + github.com/rancher/shepherd v0.0.0-20240412143253-50c4247b5902 go.qase.io/client v0.0.0-20231114201952-65195ec001fa ) diff --git a/go.sum b/go.sum index 91408277a43..ba1365febc9 100644 --- a/go.sum +++ b/go.sum @@ -1641,8 +1641,8 @@ github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcp github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= github.com/rancher/rke v1.5.9-rc1 h1:xlXoYDK9fWynUtKNP4WOiFI6ufKIBxjK7FVqk96U76A= github.com/rancher/rke v1.5.9-rc1/go.mod h1:vojhOf8U8VCmw7y17OENWXSIfEFPEbXCMQcmI7xN7i8= -github.com/rancher/shepherd v0.0.0-20240408151625-d0c3b8dbe5dd h1:Nog4ViMD04zg6GP+5IvsthOIWqps7m6mdMnhIwWkPxA= -github.com/rancher/shepherd v0.0.0-20240408151625-d0c3b8dbe5dd/go.mod h1:LNI7nH1BptYMvJmuqsLgmkMytGBBTpW4jk4vAHCxfF4= +github.com/rancher/shepherd v0.0.0-20240412143253-50c4247b5902 h1:Bs+tXgtduRRtr50SFJ1lBgLATmj4xjc6RjuS2m47Iyg= +github.com/rancher/shepherd v0.0.0-20240412143253-50c4247b5902/go.mod h1:LNI7nH1BptYMvJmuqsLgmkMytGBBTpW4jk4vAHCxfF4= github.com/rancher/steve v0.0.0-20240305150728-3943409601f1 h1:6wNYy3q9jget45syTN6K2uOLSYaptLYCHscY2WRmhDI= github.com/rancher/steve v0.0.0-20240305150728-3943409601f1/go.mod h1:o4vLBzMTKbHHhIiAcbgOiaN3aK1vIjL6ZTgaGxQYpsY= github.com/rancher/system-upgrade-controller/pkg/apis v0.0.0-20210727200656-10b094e30007 h1:ru+mqGnxMmKeU0Q3XIDxkARvInDIqT1hH2amTcsjxI4= diff --git a/tests/v2/integration/catalogv2/rancher_managed_charts_test.go b/tests/v2/integration/catalogv2/rancher_managed_charts_test.go index f4a845f53b7..b8c0d6ed24b 100644 --- a/tests/v2/integration/catalogv2/rancher_managed_charts_test.go +++ b/tests/v2/integration/catalogv2/rancher_managed_charts_test.go @@ -500,11 +500,11 @@ func (w *RancherManagedChartsTest) TestServeIcons() { w.Assert().Equal("bundled", systemCatalogUpdated.Value) // Fetch one icon with https:// scheme, it should return an empty object (i.e length of image equals 0) with nil error - imgLength, err := w.catalogClient.FetchChartIcon(smallForkClusterRepoName, "fleet", "102.0.0+up0.6.0") + imgLength, err := w.catalogClient.FetchChartIcon(smallForkClusterRepoName, "fleet") w.Require().NoError(err) w.Assert().Equal(0, imgLength) - imgLength, err = w.catalogClient.FetchChartIcon(smallForkClusterRepoName, "rancher-cis-benchmark", "4.0.0") + imgLength, err = w.catalogClient.FetchChartIcon(smallForkClusterRepoName, "rancher-cis-benchmark") w.Require().NoError(err) w.Assert().Greater(imgLength, 0) diff --git a/tests/v2/validation/provisioning/k3s/README.md b/tests/v2/validation/provisioning/k3s/README.md index 2f81cdea54b..dd0ccc4121f 100644 --- a/tests/v2/validation/provisioning/k3s/README.md +++ b/tests/v2/validation/provisioning/k3s/README.md @@ -216,7 +216,8 @@ harvesterMachineConfig": vmwarevsphereMachineConfigs: datacenter: "/" #required hostSystem: "//path-to-host" #required - datastore: "//path-to-datastore" #required + datastoreURL: "/datastore.URL" #required + datastore: ""//path-to-datastore" #required folder: "//path-to-vm-folder" #required pool: "//path-to-resource-pool" #required vmwarevsphereMachineConfig: diff --git a/tests/v2/validation/provisioning/permutations/permutations.go b/tests/v2/validation/provisioning/permutations/permutations.go index b91aea9bc0d..6f7520d9f0d 100644 --- a/tests/v2/validation/provisioning/permutations/permutations.go +++ b/tests/v2/validation/provisioning/permutations/permutations.go @@ -1,21 +1,29 @@ package permutations import ( + "context" "os" "strings" "testing" + "time" + "github.com/rancher/rancher/pkg/api/scheme" provv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1" "github.com/rancher/shepherd/clients/corral" "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" steveV1 "github.com/rancher/shepherd/clients/rancher/v1" "github.com/rancher/shepherd/extensions/charts" "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/kubeapi/storageclasses" + "github.com/rancher/shepherd/extensions/kubeapi/volumes/persistentvolumeclaims" "github.com/rancher/shepherd/extensions/machinepools" "github.com/rancher/shepherd/extensions/projects" "github.com/rancher/shepherd/extensions/provisioning" "github.com/rancher/shepherd/extensions/provisioninginput" "github.com/rancher/shepherd/extensions/rke1/componentchecks" + "github.com/rancher/shepherd/extensions/rke1/nodetemplates" "github.com/rancher/shepherd/extensions/services" "github.com/rancher/shepherd/extensions/workloads" "github.com/rancher/shepherd/extensions/workloads/pods" @@ -26,6 +34,9 @@ import ( "github.com/stretchr/testify/suite" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -47,6 +58,9 @@ const ( nginxName = "nginx" defaultNamespace = "default" + pollInterval = time.Duration(1 * time.Second) + pollTimeout = time.Duration(1 * time.Minute) + repoType = "catalog.cattle.io.clusterrepo" appsType = "catalog.cattle.io.apps" awsUpstreamCloudProviderRepo = "https://github.com/kubernetes/cloud-provider-aws.git" @@ -55,6 +69,13 @@ const ( kubeSystemNamespace = "kube-system" systemProject = "System" externalProviderString = "external" + vsphereCPIchartName = "rancher-vsphere-cpi" + vsphereCSIchartName = "rancher-vsphere-csi" +) + +var ( + group int64 + user int64 ) // RunTestPermutations runs through all relevant perumutations in a given config file, including node providers, k8s versions, and CNIs @@ -76,14 +97,24 @@ func RunTestPermutations(s *suite.Suite, testNamePrefix string, client *rancher. } else { providers = provisioningConfig.Providers } + for _, nodeProviderName := range providers { + nodeProvider, rke1Provider, customProvider, kubeVersions := GetClusterProvider(clusterType, nodeProviderName, provisioningConfig) + for _, kubeVersion := range kubeVersions { for _, cni := range provisioningConfig.CNIs { + testClusterConfig = clusters.ConvertConfigToClusterConfig(provisioningConfig) testClusterConfig.CNI = cni name = testNamePrefix + " Node Provider: " + nodeProviderName + " Kubernetes version: " + kubeVersion + " cni: " + cni + + clusterObject := &steveV1.SteveAPIObject{} + rke1ClusterObject := &management.Cluster{} + nodeTemplate := &nodetemplates.NodeTemplate{} + s.Run(name, func() { + if testClusterConfig.CloudProvider == provisioninginput.AWSProviderName.String() { byteYaml, err := os.ReadFile(outOfTreeAWSFilePath) require.NoError(s.T(), err) @@ -95,92 +126,66 @@ func RunTestPermutations(s *suite.Suite, testNamePrefix string, client *rancher. switch clusterType { case RKE2ProvisionCluster, K3SProvisionCluster: testClusterConfig.KubernetesVersion = kubeVersion - clusterObject, err := provisioning.CreateProvisioningCluster(client, *nodeProvider, testClusterConfig, hostnameTruncation) + clusterObject, err = provisioning.CreateProvisioningCluster(client, *nodeProvider, testClusterConfig, hostnameTruncation) require.NoError(s.T(), err) provisioning.VerifyCluster(s.T(), client, testClusterConfig, clusterObject) - if testClusterConfig.CloudProvider == provisioninginput.AWSProviderName.String() { - lbServiceResp := CreateCloudProviderWorkloadAndServicesLB(s.T(), client, clusterObject) - - status := &provv1.ClusterStatus{} - err := steveV1.ConvertToK8sType(clusterObject.Status, status) - require.NoError(s.T(), err) - - services.VerifyAWSLoadBalancer(s.T(), client, lbServiceResp, status.ClusterName) - } - case RKE1ProvisionCluster: testClusterConfig.KubernetesVersion = kubeVersion - nodeTemplate, err := rke1Provider.NodeTemplateFunc(client) + nodeTemplate, err = rke1Provider.NodeTemplateFunc(client) require.NoError(s.T(), err) + // workaround to simplify config for rke1 clusters with cloud provider set. This will allow external charts to be installed + // while using the rke2 CloudProvider. + if testClusterConfig.CloudProvider == provisioninginput.VsphereCloudProviderName.String() { + testClusterConfig.CloudProvider = "external" + } - clusterObject, err := provisioning.CreateProvisioningRKE1Cluster(client, *rke1Provider, testClusterConfig, nodeTemplate) + rke1ClusterObject, err = provisioning.CreateProvisioningRKE1Cluster(client, *rke1Provider, testClusterConfig, nodeTemplate) require.NoError(s.T(), err) - provisioning.VerifyRKE1Cluster(s.T(), client, testClusterConfig, clusterObject) - - if strings.Contains(testClusterConfig.CloudProvider, provisioninginput.AWSProviderName.String()) { - if strings.Contains(testClusterConfig.CloudProvider, externalProviderString) { - err = CreateAndInstallAWSExternalCharts(client, clusterObject.ID, false) - require.NoError(s.T(), err) - - podErrors := pods.StatusPods(client, clusterObject.ID) - require.Empty(s.T(), podErrors) - } - - adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) - require.NoError(s.T(), err) - - steveClusterObject, err := adminClient.Steve.SteveType(clusters.ProvisioningSteveResourceType).ByID(provisioninginput.Namespace + "/" + clusterObject.ID) - require.NoError(s.T(), err) - - lbServiceResp := CreateCloudProviderWorkloadAndServicesLB(s.T(), client, steveClusterObject) - - status := &provv1.ClusterStatus{} - err = steveV1.ConvertToK8sType(steveClusterObject.Status, status) - require.NoError(s.T(), err) - - services.VerifyAWSLoadBalancer(s.T(), client, lbServiceResp, status.ClusterName) - } + provisioning.VerifyRKE1Cluster(s.T(), client, testClusterConfig, rke1ClusterObject) case RKE2CustomCluster, K3SCustomCluster: testClusterConfig.KubernetesVersion = kubeVersion - clusterObject, err := provisioning.CreateProvisioningCustomCluster(client, customProvider, testClusterConfig) + clusterObject, err = provisioning.CreateProvisioningCustomCluster(client, customProvider, testClusterConfig) require.NoError(s.T(), err) provisioning.VerifyCluster(s.T(), client, testClusterConfig, clusterObject) - if testClusterConfig.CloudProvider == provisioninginput.AWSProviderName.String() { - lbServiceResp := CreateCloudProviderWorkloadAndServicesLB(s.T(), client, clusterObject) - - status := &provv1.ClusterStatus{} - err := steveV1.ConvertToK8sType(clusterObject.Status, status) - require.NoError(s.T(), err) - - services.VerifyAWSLoadBalancer(s.T(), client, lbServiceResp, status.ClusterName) - } case RKE1CustomCluster: testClusterConfig.KubernetesVersion = kubeVersion - clusterObject, nodes, err := provisioning.CreateProvisioningRKE1CustomCluster(client, customProvider, testClusterConfig) + // workaround to simplify config for rke1 clusters with cloud provider set. This will allow external charts to be installed + // while using the rke2 CloudProvider name in the + if testClusterConfig.CloudProvider == provisioninginput.VsphereCloudProviderName.String() { + testClusterConfig.CloudProvider = "external" + } + + rke1ClusterObject, nodes, err := provisioning.CreateProvisioningRKE1CustomCluster(client, customProvider, testClusterConfig) require.NoError(s.T(), err) - provisioning.VerifyRKE1Cluster(s.T(), client, testClusterConfig, clusterObject) - etcdVersion, err := componentchecks.CheckETCDVersion(client, nodes, clusterObject.ID) + provisioning.VerifyRKE1Cluster(s.T(), client, testClusterConfig, rke1ClusterObject) + etcdVersion, err := componentchecks.CheckETCDVersion(client, nodes, rke1ClusterObject.ID) require.NoError(s.T(), err) require.NotEmpty(s.T(), etcdVersion) // airgap currently uses corral to create nodes and register with rancher case RKE2AirgapCluster, K3SAirgapCluster: testClusterConfig.KubernetesVersion = kubeVersion - clusterObject, err := provisioning.CreateProvisioningAirgapCustomCluster(client, testClusterConfig, corralPackages) + clusterObject, err = provisioning.CreateProvisioningAirgapCustomCluster(client, testClusterConfig, corralPackages) require.NoError(s.T(), err) provisioning.VerifyCluster(s.T(), client, testClusterConfig, clusterObject) case RKE1AirgapCluster: testClusterConfig.KubernetesVersion = kubeVersion + // workaround to simplify config for rke1 clusters with cloud provider set. This will allow external charts to be installed + // while using the rke2 CloudProvider name in the + if testClusterConfig.CloudProvider == provisioninginput.VsphereCloudProviderName.String() { + testClusterConfig.CloudProvider = "external" + } + clusterObject, err := provisioning.CreateProvisioningRKE1AirgapCustomCluster(client, testClusterConfig, corralPackages) require.NoError(s.T(), err) @@ -190,12 +195,76 @@ func RunTestPermutations(s *suite.Suite, testNamePrefix string, client *rancher. s.T().Fatalf("Invalid cluster type: %s", clusterType) } + RunPostClusterCloudProviderChecks(s.T(), client, clusterType, nodeTemplate, testClusterConfig, clusterObject, rke1ClusterObject) + }) } } } } +// RunPostClusterCloudProviderChecks does additinal checks on the cluster if there's a cloud provider set +// on an active cluster. +func RunPostClusterCloudProviderChecks(t *testing.T, client *rancher.Client, clusterType string, nodeTemplate *nodetemplates.NodeTemplate, testClusterConfig *clusters.ClusterConfig, clusterObject *steveV1.SteveAPIObject, rke1ClusterObject *management.Cluster) { + if strings.Contains(clusterType, clusters.RKE1ClusterType.String()) { + providers := *testClusterConfig.Providers + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + require.NoError(t, err) + + if strings.Contains(testClusterConfig.CloudProvider, provisioninginput.AWSProviderName.String()) { + if strings.Contains(testClusterConfig.CloudProvider, externalProviderString) { + err := CreateAndInstallAWSExternalCharts(client, rke1ClusterObject.ID, false) + require.NoError(t, err) + + podErrors := pods.StatusPods(client, rke1ClusterObject.ID) + require.Empty(t, podErrors) + } + + clusterObject, err = adminClient.Steve.SteveType(clusters.ProvisioningSteveResourceType).ByID(provisioninginput.Namespace + "/" + rke1ClusterObject.ID) + require.NoError(t, err) + + lbServiceResp := CreateCloudProviderWorkloadAndServicesLB(t, client, clusterObject) + + status := &provv1.ClusterStatus{} + err = steveV1.ConvertToK8sType(clusterObject.Status, status) + require.NoError(t, err) + + services.VerifyAWSLoadBalancer(t, client, lbServiceResp, status.ClusterName) + } else if strings.Contains(testClusterConfig.CloudProvider, "external") && providers[0] == provisioninginput.VsphereProviderName.String() { + err := charts.InstallVsphereOutOfTreeCharts(client, nodeTemplate, catalog.RancherChartRepo, rke1ClusterObject.ID) + require.NoError(t, err) + + podErrors := pods.StatusPods(client, rke1ClusterObject.ID) + require.Empty(t, podErrors) + + clusterObject, err := adminClient.Steve.SteveType(clusters.ProvisioningSteveResourceType).ByID(provisioninginput.Namespace + "/" + rke1ClusterObject.ID) + require.NoError(t, err) + + CreatePVCWorkload(t, client, clusterObject) + } + } else if strings.Contains(clusterType, clusters.RKE2ClusterType.String()) { + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + require.NoError(t, err) + + if testClusterConfig.CloudProvider == provisioninginput.AWSProviderName.String() { + clusterObject, err := adminClient.Steve.SteveType(clusters.ProvisioningSteveResourceType).ByID(clusterObject.ID) + require.NoError(t, err) + + lbServiceResp := CreateCloudProviderWorkloadAndServicesLB(t, client, clusterObject) + + status := &provv1.ClusterStatus{} + err = steveV1.ConvertToK8sType(clusterObject.Status, status) + require.NoError(t, err) + + services.VerifyAWSLoadBalancer(t, client, lbServiceResp, status.ClusterName) + } + + if testClusterConfig.CloudProvider == provisioninginput.VsphereCloudProviderName.String() { + CreatePVCWorkload(t, client, clusterObject) + } + } +} + // GetClusterProvider returns a provider object given cluster type, nodeProviderName (for custom clusters) and the provisioningConfig func GetClusterProvider(clusterType string, nodeProviderName string, provisioningConfig *provisioninginput.Config) (*provisioning.Provider, *provisioning.RKE1Provider, *provisioning.ExternalNodeProvider, []string) { var nodeProvider provisioning.Provider @@ -238,6 +307,7 @@ func GetClusterProvider(clusterType string, nodeProviderName string, provisionin // This should be used when testing cloud provider with in-tree or out-of-tree set on the cluster. func CreateCloudProviderWorkloadAndServicesLB(t *testing.T, client *rancher.Client, cluster *steveV1.SteveAPIObject) *steveV1.SteveAPIObject { status := &provv1.ClusterStatus{} + err := steveV1.ConvertToK8sType(cluster.Status, status) require.NoError(t, err) @@ -268,6 +338,112 @@ func CreateCloudProviderWorkloadAndServicesLB(t *testing.T, client *rancher.Clie return lbServiceResp } +// CreatePVCWorkload creates a workload with a PVC for storage. This helper should be used to test +// storage class functionality, i.e. for an in-tree / out-of-tree cloud provider +func CreatePVCWorkload(t *testing.T, client *rancher.Client, cluster *steveV1.SteveAPIObject) *steveV1.SteveAPIObject { + + status := &provv1.ClusterStatus{} + err := steveV1.ConvertToK8sType(cluster.Status, status) + require.NoError(t, err) + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + require.NoError(t, err) + + steveclient, err := adminClient.Steve.ProxyDownstream(status.ClusterName) + require.NoError(t, err) + + dynamicClient, err := client.GetDownStreamClusterClient(status.ClusterName) + require.NoError(t, err) + + storageClassVolumesResource := dynamicClient.Resource(storageclasses.StorageClassGroupVersionResource).Namespace("") + + ctx := context.Background() + unstructuredResp, err := storageClassVolumesResource.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + + storageClasses := &v1.StorageClassList{} + + err = scheme.Scheme.Convert(unstructuredResp, storageClasses, unstructuredResp.GroupVersionKind()) + require.NoError(t, err) + + storageClass := storageClasses.Items[0] + + logrus.Infof("creating PVC") + + accessModes := []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + } + + persistentVolumeClaim, err := persistentvolumeclaims.CreatePersistentVolumeClaim( + client, + status.ClusterName, + namegenerator.AppendRandomString("pvc"), + "test-pvc-volume", + defaultNamespace, + 1, + accessModes, + nil, + &storageClass, + ) + require.NoError(t, err) + + pvcStatus := &corev1.PersistentVolumeClaimStatus{} + stevePvc := &steveV1.SteveAPIObject{} + + err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (done bool, err error) { + + stevePvc, err = steveclient.SteveType(persistentvolumeclaims.PersistentVolumeClaimType).ByID(defaultNamespace + "/" + persistentVolumeClaim.Name) + require.NoError(t, err) + + err = steveV1.ConvertToK8sType(stevePvc.Status, pvcStatus) + require.NoError(t, err) + + if pvcStatus.Phase == persistentvolumeclaims.PersistentVolumeBoundStatus { + return true, nil + } + return false, err + }) + require.NoError(t, err) + + nginxResponse, err := createNginxDeploymentWithPVC(steveclient, "pvcwkld", persistentVolumeClaim.Name, string(stevePvc.Spec.(map[string]interface{})[persistentvolumeclaims.StevePersistentVolumeClaimVolumeName].(string))) + require.NoError(t, err) + + return nginxResponse +} + +// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace +func createNginxDeploymentWithPVC(steveclient *steveV1.Client, containerNamePrefix, pvcName, volName string) (*steveV1.SteveAPIObject, error) { + + logrus.Infof("Vol: %s", volName) + logrus.Infof("Pod: %s", pvcName) + + containerName := namegenerator.AppendRandomString(containerNamePrefix) + volMount := *&corev1.VolumeMount{ + MountPath: "/auto-mnt", + Name: volName, + } + + podVol := corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + } + + containerTemplate := workloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{volMount}, []corev1.EnvFromSource{}, nil, nil, nil) + podTemplate := workloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, nil) + deployment := workloads.NewDeploymentTemplate(containerName, defaultNamespace, podTemplate, true, nil) + + deploymentResp, err := steveclient.SteveType(workloads.DeploymentSteveType).Create(deployment) + if err != nil { + return nil, err + } + + return deploymentResp, err +} + // createNginxDeployment is a helper function that creates a nginx deployment in a cluster's default namespace func createNginxDeployment(steveclient *steveV1.Client, containerNamePrefix string) (*steveV1.SteveAPIObject, error) { containerName := namegenerator.AppendRandomString(containerNamePrefix) diff --git a/tests/v2/validation/provisioning/rke1/README.md b/tests/v2/validation/provisioning/rke1/README.md index 7449bc363e1..19e314800f5 100644 --- a/tests/v2/validation/provisioning/rke1/README.md +++ b/tests/v2/validation/provisioning/rke1/README.md @@ -68,6 +68,19 @@ provisioningInput: snapshot: false ``` + +## Cloud Provider +Cloud Provider enables additional options through the cloud provider, like cloud persistent storage or cloud provisioned load balancers. + +Names of cloud provider options are typically controlled by rancher product. Hence the discrepancy in rke2 vs. rke1 AWS in-tree and out-of-tree options. +To use automation with a cloud provider, simply enter one of the following options in the `cloudProvider` field in the config. + +### RKE1 Cloud Provider Options +* external-aws +* aws +* rancher-vsphere + + ## NodeTemplateConfigs RKE1 specifically needs a node template config to run properly. These are the inputs needed for the different node providers. Top level node template config entries can be set. The top level nodeTemplate is optional, and is not need for the different node @@ -209,6 +222,7 @@ vmwarevsphereNodeConfig: creationType: "" datacenter: "" datastore: "" + datastoreURL: "" datastoreCluster: "" diskSize: "20000" folder: "" diff --git a/tests/v2/validation/provisioning/rke1/provisioning_cloud_provider_test.go b/tests/v2/validation/provisioning/rke1/provisioning_cloud_provider_test.go index 98144100cde..4832e747da3 100644 --- a/tests/v2/validation/provisioning/rke1/provisioning_cloud_provider_test.go +++ b/tests/v2/validation/provisioning/rke1/provisioning_cloud_provider_test.go @@ -3,6 +3,7 @@ package rke1 import ( + "slices" "testing" "github.com/rancher/rancher/tests/v2/validation/provisioning/permutations" @@ -84,6 +85,11 @@ func (r *RKE1CloudProviderTestSuite) TestAWSCloudProviderRKE1Cluster() { }{ {"OutOfTree" + provisioninginput.StandardClientName.String(), nodeRolesDedicated, r.standardUserClient, r.client.Flags.GetValue(environmentflag.Long)}, } + + if !slices.Contains(r.provisioningConfig.Providers, "aws") { + r.T().Skip("AWS Cloud Provider test requires access to aws.") + } + for _, tt := range tests { if !tt.runFlag { r.T().Logf("SKIPPED") @@ -96,6 +102,37 @@ func (r *RKE1CloudProviderTestSuite) TestAWSCloudProviderRKE1Cluster() { } } +func (r *RKE1CloudProviderTestSuite) TestVsphereCloudProviderRKE1Cluster() { + nodeRolesDedicated := []provisioninginput.NodePools{provisioninginput.EtcdNodePool, provisioninginput.ControlPlaneNodePool, provisioninginput.WorkerNodePool} + nodeRolesDedicated[0].NodeRoles.Quantity = 3 + nodeRolesDedicated[1].NodeRoles.Quantity = 2 + nodeRolesDedicated[2].NodeRoles.Quantity = 2 + + tests := []struct { + name string + nodePools []provisioninginput.NodePools + client *rancher.Client + runFlag bool + }{ + {"OutOfTree" + provisioninginput.StandardClientName.String(), nodeRolesDedicated, r.standardUserClient, r.client.Flags.GetValue(environmentflag.Long)}, + } + + if !slices.Contains(r.provisioningConfig.Providers, "vsphere") { + r.T().Skip("Vsphere Cloud Provider test requires access to Vsphere.") + } + + for _, tt := range tests { + if !tt.runFlag { + r.T().Logf("SKIPPED") + continue + } + provisioningConfig := *r.provisioningConfig + provisioningConfig.CloudProvider = "rancher-vsphere" + provisioningConfig.NodePools = tt.nodePools + permutations.RunTestPermutations(&r.Suite, tt.name, tt.client, &provisioningConfig, permutations.RKE1ProvisionCluster, nil, nil) + } +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestRKE1CloudProviderTestSuite(t *testing.T) { diff --git a/tests/v2/validation/provisioning/rke2/README.md b/tests/v2/validation/provisioning/rke2/README.md index d36b35516c4..8872de0675c 100644 --- a/tests/v2/validation/provisioning/rke2/README.md +++ b/tests/v2/validation/provisioning/rke2/README.md @@ -123,11 +123,15 @@ vmwarevsphereCredentials: ``` ## Cloud Provider -Cloud Provider enables additional options such as load-balancers and storage devices to be provisioned through your cluster -available options: -### AWS +Cloud Provider enables additional options through the cloud provider, like cloud persistent storage or cloud provisioned load balancers. + +Names of cloud provider options are typically controlled by rancher product. Hence the discrepancy in rke2 vs. rke1 AWS in-tree and out-of-tree options. +To use automation with a cloud provider, simply enter one of the following options in the `cloudProvider` field in the config. + +### RKE2 Cloud Provider Options * `aws-in-tree` uses the in-tree provider for aws -- **Deprecated on kubernetes 1.26 and below** * `aws` uses the out-of-tree provider for aws. Built in logic to the automation will be applied to the cluster that applies the correct configuration for the out-of-tree charts to be installed. Supported on kubernetes 1.22+ +* rancher-vsphere ## Machine RKE2 Config Machine RKE2 config is the final piece needed for the config to run RKE2 provisioning tests. @@ -233,6 +237,7 @@ vmwarevsphereMachineConfigs: datacenter: "/" #required hostSystem: "//path-to-host" #required datastore: "//path-to-datastore" #required + datastoreURL: "ds:///" folder: "//path-to-vm-folder" #required pool: "//path-to-resource-pool" #required vmwarevsphereMachineConfig: diff --git a/tests/v2/validation/provisioning/rke2/provisioning_cloud_provider_test.go b/tests/v2/validation/provisioning/rke2/provisioning_cloud_provider_test.go index 47a070385ff..4fb11aa591b 100644 --- a/tests/v2/validation/provisioning/rke2/provisioning_cloud_provider_test.go +++ b/tests/v2/validation/provisioning/rke2/provisioning_cloud_provider_test.go @@ -3,6 +3,7 @@ package rke2 import ( + "slices" "testing" "github.com/rancher/rancher/tests/v2/validation/provisioning/permutations" @@ -83,11 +84,16 @@ func (r *RKE2CloudProviderTestSuite) TestAWSCloudProviderCluster() { {"OutOfTree" + provisioninginput.StandardClientName.String(), nodeRolesDedicated, r.standardUserClient, r.client.Flags.GetValue(environmentflag.Long)}, } + if !slices.Contains(r.provisioningConfig.Providers, "aws") { + r.T().Skip("AWS Cloud Provider test requires access to aws.") + } + for _, tt := range tests { if !tt.runFlag { r.T().Logf("SKIPPED") continue } + provisioningConfig := *r.provisioningConfig provisioningConfig.CloudProvider = "aws" provisioningConfig.MachinePools = tt.machinePools @@ -95,6 +101,38 @@ func (r *RKE2CloudProviderTestSuite) TestAWSCloudProviderCluster() { } } +func (r *RKE2CloudProviderTestSuite) TestVsphereCloudProviderCluster() { + nodeRolesDedicated := []provisioninginput.MachinePools{provisioninginput.EtcdMachinePool, provisioninginput.ControlPlaneMachinePool, provisioninginput.WorkerMachinePool} + nodeRolesDedicated[0].MachinePoolConfig.Quantity = 3 + nodeRolesDedicated[1].MachinePoolConfig.Quantity = 2 + nodeRolesDedicated[2].MachinePoolConfig.Quantity = 2 + + tests := []struct { + name string + machinePools []provisioninginput.MachinePools + client *rancher.Client + runFlag bool + }{ + {"OutOfTree" + provisioninginput.StandardClientName.String(), nodeRolesDedicated, r.standardUserClient, r.client.Flags.GetValue(environmentflag.Long)}, + } + + if !slices.Contains(r.provisioningConfig.Providers, "vsphere") { + r.T().Skip("Vsphere Cloud Provider test requires access to vsphere.") + } + + for _, tt := range tests { + if !tt.runFlag { + r.T().Logf("SKIPPED") + continue + } + + provisioningConfig := *r.provisioningConfig + provisioningConfig.CloudProvider = "rancher-vsphere" + provisioningConfig.MachinePools = tt.machinePools + permutations.RunTestPermutations(&r.Suite, tt.name, tt.client, &provisioningConfig, permutations.RKE2ProvisionCluster, nil, nil) + } +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestRKE2CloudProviderTestSuite(t *testing.T) {