From bed2ab86ba810d4d1db642ed4c286330c9d502f1 Mon Sep 17 00:00:00 2001 From: Robert Nemeti Date: Wed, 3 Jul 2024 14:23:32 +0200 Subject: [PATCH 1/4] K8S-10217 fix waiting for cluster update the status of the control plane components is flipped by a controller which may not be quick enough before terraform checks it --- metakube/resource_cluster.go | 28 +++++++++++------------- metakube/resource_maintenance_cronjob.go | 2 +- metakube/resource_node_deployment.go | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/metakube/resource_cluster.go b/metakube/resource_cluster.go index 44de543..08cc6d5 100644 --- a/metakube/resource_cluster.go +++ b/metakube/resource_cluster.go @@ -201,7 +201,7 @@ func metakubeResourceClusterCreate(ctx context.Context, d *schema.ResourceData, return diag.FromErr(err) } - if err := metakubeResourceClusterWaitForReady(ctx, meta, d.Timeout(schema.TimeoutCreate), projectID, d.Id()); err != nil { + if err := metakubeResourceClusterWaitForReady(ctx, meta, d.Timeout(schema.TimeoutCreate), projectID, d.Id(), ""); err != nil { // In case of timeout, we still want to return the cluster resource retDiags = append(retDiags, metakubeResourceClusterRead(ctx, d, m)...) retDiags = append(retDiags, diag.Diagnostic{ @@ -610,7 +610,11 @@ func metakubeResourceClusterUpdate(ctx context.Context, d *schema.ResourceData, } } - if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutUpdate), projectID, d.Id()); err != nil { + configuredVersion, ok := d.GetOk("spec.0.version") + if !ok { + return nil + } + if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutUpdate), projectID, d.Id(), configuredVersion.(string)); err != nil { return diag.Errorf("cluster '%s' is not ready: %v", d.Id(), err) } @@ -736,28 +740,22 @@ func assignSSHKeysToCluster(projectID, clusterID string, sshkeyIDs []string, k * return nil } -func metakubeResourceClusterWaitForReady(ctx context.Context, k *metakubeProviderMeta, timeout time.Duration, projectID, clusterID string) error { +func metakubeResourceClusterWaitForReady(ctx context.Context, k *metakubeProviderMeta, timeout time.Duration, projectID, clusterID, configuredVersion string) error { return retry.RetryContext(ctx, timeout, func() *retry.RetryError { - p := project.NewGetClusterHealthV2Params() + p := project.NewGetClusterV2Params() p.SetContext(ctx) p.SetProjectID(projectID) p.SetClusterID(clusterID) - r, err := k.client.Project.GetClusterHealthV2(p, k.auth) + r, err := k.client.Project.GetClusterV2(p, k.auth) if err != nil { - return retry.RetryableError(fmt.Errorf("unable to get cluster '%s' health: %s", clusterID, stringifyResponseError(err))) + return retry.RetryableError(fmt.Errorf("unable to get cluster '%s': %s", clusterID, stringifyResponseError(err))) } - const up models.HealthStatus = 1 - - if r.Payload.Apiserver == up && - r.Payload.CloudProviderInfrastructure == up && - r.Payload.Controller == up && - r.Payload.Etcd == up && - r.Payload.MachineController == up && - r.Payload.Scheduler == up && - r.Payload.UserClusterControllerManager == up { + if configuredVersion == "" { + return nil + } else if r.Payload.Status.Version == models.Semver(configuredVersion) { return nil } diff --git a/metakube/resource_maintenance_cronjob.go b/metakube/resource_maintenance_cronjob.go index f7cf903..9a3b741 100644 --- a/metakube/resource_maintenance_cronjob.go +++ b/metakube/resource_maintenance_cronjob.go @@ -101,7 +101,7 @@ func metakubeResourceMaintenanceCronJobCreate(ctx context.Context, d *schema.Res } // TODO not sure to remove this - if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID); err != nil { + if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID, ""); err != nil { return diag.Errorf("cluster is not ready: %v", err) } diff --git a/metakube/resource_node_deployment.go b/metakube/resource_node_deployment.go index 4e4c6fd..c0a8e3a 100644 --- a/metakube/resource_node_deployment.go +++ b/metakube/resource_node_deployment.go @@ -124,7 +124,7 @@ func metakubeResourceNodeDeploymentCreate(ctx context.Context, d *schema.Resourc return diag.FromErr(err) } - if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID); err != nil { + if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID, ""); err != nil { return diag.Errorf("cluster is not ready: %v", err) } From 9f21856b99f35a3a483caefae9f5ba42017fbc35 Mon Sep 17 00:00:00 2001 From: Robert Nemeti Date: Thu, 4 Jul 2024 15:19:10 +0200 Subject: [PATCH 2/4] replace openstack user credentials with application credentials in cluster and machinedeployment tests --- metakube/provider_test.go | 6 +- metakube/resource_cluster_test.go | 136 +++++++++++----------- metakube/resource_node_deployment_test.go | 47 ++++---- 3 files changed, 92 insertions(+), 97 deletions(-) diff --git a/metakube/provider_test.go b/metakube/provider_test.go index a496a85..82631c3 100644 --- a/metakube/provider_test.go +++ b/metakube/provider_test.go @@ -27,9 +27,7 @@ const ( testEnvOpenstackNodeDC = "METAKUBE_OPENSTACK_NODE_DC" testEnvOpenstackApplicationCredentialsID = "METAKUBE_OPENSTACK_APPLICATION_CREDENTIALS_ID" testEnvOpenstackApplicationCredentialsSecret = "METAKUBE_OPENSTACK_APPLICATION_CREDENTIALS_SECRET" - testEnvOpenstackUsername = "METAKUBE_OPENSTACK_USERNAME" testEnvOpenstackAuthURL = "METAKUBE_OPENSTACK_AUTH_URL" - testEnvOpenstackPassword = "METAKUBE_OPENSTACK_PASSWORD" testEnvOpenstackProjectID = "METAKUBE_OPENSTACK_PROJECT_ID" testEnvOpenstackProjectName = "METAKUBE_OPENSTACK_PROJECT_NAME" testEnvOpenstackRegion = "METAKUBE_OPENSTACK_REGION" @@ -81,8 +79,8 @@ func testAccPreCheckForOpenstack(t *testing.T) { t.Helper() testAccPreCheck(t) checkEnv(t, testEnvK8sVersionOpenstack) - checkEnv(t, testEnvOpenstackUsername) - checkEnv(t, testEnvOpenstackPassword) + checkEnv(t, testEnvOpenstackApplicationCredentialsID) + checkEnv(t, testEnvOpenstackApplicationCredentialsSecret) checkEnv(t, testEnvOpenstackProjectID) checkEnv(t, testEnvOpenstackProjectName) checkEnv(t, testEnvOpenstackRegion) diff --git a/metakube/resource_cluster_test.go b/metakube/resource_cluster_test.go index 9b5240d..08621c9 100644 --- a/metakube/resource_cluster_test.go +++ b/metakube/resource_cluster_test.go @@ -51,15 +51,15 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) { resourceName := "metakube_cluster.acctest_cluster" data := &clusterOpenstackBasicData{ - Name: makeRandomName() + "-basic", - OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), - OpenstackUser: os.Getenv(testEnvOpenstackUsername), - OpenstackPassword: os.Getenv(testEnvOpenstackPassword), - OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), - OpenstackRegion: os.Getenv(testEnvOpenstackRegion), - DatacenterName: os.Getenv(testEnvOpenstackNodeDC), - ProjectID: os.Getenv(testEnvProjectID), - Version: os.Getenv(testEnvK8sVersionOpenstack), + Name: makeRandomName() + "-basic", + OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), + OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), + OpenstackRegion: os.Getenv(testEnvOpenstackRegion), + DatacenterName: os.Getenv(testEnvOpenstackNodeDC), + ProjectID: os.Getenv(testEnvProjectID), + Version: os.Getenv(testEnvK8sVersionOpenstack), } var config strings.Builder if err := clusterOpenstackBasicTemplate.Execute(&config, data); err != nil { @@ -222,18 +222,18 @@ func TestAccMetakubeCluster_Openstack_ApplicationCredentials_Dynammic(t *testing var cluster models.Cluster resourceName := "metakube_cluster.acctest_cluster" data := &clusterOpenstackApplicationCredentailsData{ - Name: makeRandomName() + "-appcred-dynamic", - OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), - OpenstackUser: os.Getenv(testEnvOpenstackUsername), - OpenstackPassword: os.Getenv(testEnvOpenstackPassword), - OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), - OpenstackRegion: os.Getenv(testEnvOpenstackRegion), - DatacenterName: os.Getenv(testEnvOpenstackNodeDC), - ProjectID: os.Getenv(testEnvProjectID), - Version: os.Getenv(testEnvK8sVersionOpenstack), - OpenstackApplicationCredentialID: os.Getenv(testEnvOpenstackApplicationCredentialsID), - OpenstackApplicationCredentialSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), - Dynamic: true, + Name: makeRandomName() + "-appcred-dynamic", + OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), + OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), + OpenstackRegion: os.Getenv(testEnvOpenstackRegion), + DatacenterName: os.Getenv(testEnvOpenstackNodeDC), + ProjectID: os.Getenv(testEnvProjectID), + Version: os.Getenv(testEnvK8sVersionOpenstack), + OpenstackApplicationCredentialID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + Dynamic: true, } var config strings.Builder if err := clusterOpenstackApplicationCredentialsBasicTemplate.Execute(&config, data); err != nil { @@ -267,15 +267,15 @@ func TestAccMetakubeCluster_Openstack_UpgradeVersion(t *testing.T) { resourceName := "metakube_cluster.acctest_cluster" versionedConfig := func(version string) string { data := &clusterOpenstackBasicData{ - Name: makeRandomName() + "-upgrade", - Version: version, - OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), - OpenstackUser: os.Getenv(testEnvOpenstackUsername), - OpenstackPassword: os.Getenv(testEnvOpenstackPassword), - OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), - DatacenterName: os.Getenv(testEnvOpenstackNodeDC), - ProjectID: os.Getenv(testEnvProjectID), - OpenstackRegion: os.Getenv(testEnvOpenstackRegion), + Name: makeRandomName() + "-upgrade", + Version: version, + OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), + OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), + DatacenterName: os.Getenv(testEnvOpenstackNodeDC), + ProjectID: os.Getenv(testEnvProjectID), + OpenstackRegion: os.Getenv(testEnvOpenstackRegion), } var result strings.Builder if err := clusterOpenstackBasicTemplate.Execute(&result, data); err != nil { @@ -315,11 +315,11 @@ func TestAccMetakubeCluster_Openstack_UpgradeVersion(t *testing.T) { } type clusterOpenstackBasicData struct { - OpenstackAuthURL string - OpenstackUser string - OpenstackPassword string - OpenstackProjectID string - OpenstackRegion string + OpenstackAuthURL string + OpenstackApplicationCredentialsID string + OpenstackApplicationCredentialsSecret string + OpenstackProjectID string + OpenstackRegion string Name string DatacenterName string @@ -342,9 +342,8 @@ terraform { provider "openstack" { auth_url = "{{ .OpenstackAuthURL }}" - user_name = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" - tenant_id = "{{ .OpenstackProjectID }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" region = "{{ .OpenstackRegion }}" } @@ -372,10 +371,10 @@ resource "metakube_cluster" "acctest_cluster" { } cloud { openstack { - user_credentials { + application_credentials { project_id = "{{ .OpenstackProjectID }}" - username = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" security_group = openstack_networking_secgroup_v2.cluster-net.name @@ -426,11 +425,11 @@ resource "openstack_networking_subnet_v2" "subnet_tf_test" { }`) type clusterOpenstackApplicationCredentailsData struct { - OpenstackAuthURL string - OpenstackUser string - OpenstackPassword string - OpenstackProjectID string - OpenstackRegion string + OpenstackAuthURL string + OpenstackApplicationCredentialsID string + OpenstackApplicationCredentialsSecret string + OpenstackProjectID string + OpenstackRegion string Name string DatacenterName string @@ -453,9 +452,8 @@ terraform { {{ if .Dynamic }} provider "openstack" { auth_url = "{{ .OpenstackAuthURL }}" - user_name = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" - tenant_id = "{{ .OpenstackProjectID }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" region = "{{ .OpenstackRegion }}" } @@ -544,13 +542,13 @@ func TestAccMetakubeCluster_SSHKeys(t *testing.T) { resourceName := "metakube_cluster.acctest_cluster" data := &clusterOpenstackWithSSHKeyData{ - Name: makeRandomName() + "-sshkeys", - OpenstackUser: os.Getenv(testEnvOpenstackUsername), - OpenstackPassword: os.Getenv(testEnvOpenstackPassword), - OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), - DatacenterName: os.Getenv(testEnvOpenstackNodeDC), - ProjectID: os.Getenv(testEnvProjectID), - Version: os.Getenv(testEnvK8sVersionOpenstack), + Name: makeRandomName() + "-sshkeys", + OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), + DatacenterName: os.Getenv(testEnvOpenstackNodeDC), + ProjectID: os.Getenv(testEnvProjectID), + Version: os.Getenv(testEnvK8sVersionOpenstack), } var config1 strings.Builder @@ -591,13 +589,13 @@ func TestAccMetakubeCluster_SSHKeys(t *testing.T) { } type clusterOpenstackWithSSHKeyData struct { - Name string - DatacenterName string - ProjectID string - Version string - OpenstackProjectID string - OpenstackUser string - OpenstackPassword string + Name string + DatacenterName string + ProjectID string + Version string + OpenstackProjectID string + OpenstackApplicationCredentialsID string + OpenstackApplicationCredentialsSecret string } var clusterOpenstackTemplateWithSSHKey1 = mustParseTemplate("clusterOpenstackWithSSHKey1", ` @@ -615,10 +613,10 @@ resource "metakube_cluster" "acctest_cluster" { enable_ssh_agent = true cloud { openstack { - user_credentials { + application_credentials { project_id = "{{ .OpenstackProjectID }}" - username = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" } @@ -647,10 +645,10 @@ resource "metakube_cluster" "acctest_cluster" { enable_ssh_agent = true cloud { openstack { - user_credentials { + application_credentials { project_id = "{{ .OpenstackProjectID }}" - username = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" } diff --git a/metakube/resource_node_deployment_test.go b/metakube/resource_node_deployment_test.go index 1b990c9..bf8aa8e 100644 --- a/metakube/resource_node_deployment_test.go +++ b/metakube/resource_node_deployment_test.go @@ -20,19 +20,19 @@ func TestAccMetakubeNodeDeployment_Openstack_Basic(t *testing.T) { serverGroupResourceName := "openstack_compute_servergroup_v2.acctest_sg" data := &nodeDeploymentBasicData{ - Name: makeRandomName() + "-os-nodedepl", - OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), - OpenstackUser: os.Getenv(testEnvOpenstackUsername), - OpenstackPassword: os.Getenv(testEnvOpenstackPassword), - OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), - OpenstackRegion: os.Getenv(testEnvOpenstackRegion), - DatacenterName: os.Getenv(testEnvOpenstackNodeDC), - ProjectID: os.Getenv(testEnvProjectID), - ClusterVersion: os.Getenv(testEnvK8sVersionOpenstack), - KubeletVersion: os.Getenv(testEnvK8sOlderVersion), - NodeFlavor: os.Getenv(testEnvOpenstackFlavor), - OSVersion: os.Getenv(testEnvOpenstackImage), - UseFloatingIP: "false", + Name: makeRandomName() + "-os-nodedepl", + OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), + OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), + OpenstackRegion: os.Getenv(testEnvOpenstackRegion), + DatacenterName: os.Getenv(testEnvOpenstackNodeDC), + ProjectID: os.Getenv(testEnvProjectID), + ClusterVersion: os.Getenv(testEnvK8sVersionOpenstack), + KubeletVersion: os.Getenv(testEnvK8sOlderVersion), + NodeFlavor: os.Getenv(testEnvOpenstackFlavor), + OSVersion: os.Getenv(testEnvOpenstackImage), + UseFloatingIP: "false", } var config strings.Builder @@ -137,11 +137,11 @@ func TestAccMetakubeNodeDeployment_Openstack_Basic(t *testing.T) { } type nodeDeploymentBasicData struct { - OpenstackAuthURL string - OpenstackUser string - OpenstackPassword string - OpenstackProjectID string - OpenstackRegion string + OpenstackAuthURL string + OpenstackApplicationCredentialsID string + OpenstackApplicationCredentialsSecret string + OpenstackProjectID string + OpenstackRegion string Name string DatacenterName string @@ -165,9 +165,8 @@ var nodeDeploymentBasicTemplate = mustParseTemplate("nodeDeploymentBasic", ` } provider "openstack" { auth_url = "{{ .OpenstackAuthURL }}" - user_name = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" - tenant_id = "{{ .OpenstackProjectID }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" region = "{{ .OpenstackRegion }}" } @@ -189,10 +188,10 @@ var nodeDeploymentBasicTemplate = mustParseTemplate("nodeDeploymentBasic", ` version = "{{ .ClusterVersion }}" cloud { openstack { - user_credentials { + application_credentials { project_id = "{{ .OpenstackProjectID }}" - username = "{{ .OpenstackUser }}" - password = "{{ .OpenstackPassword }}" + application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" + application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" } From 686c8975d16082d5ca257bce0174fb70bde7fcf8 Mon Sep 17 00:00:00 2001 From: Robert Nemeti Date: Thu, 4 Jul 2024 17:38:59 +0200 Subject: [PATCH 3/4] remove project_id from application credentials --- metakube/resource_cluster_role_binding_test.go | 2 +- metakube/resource_cluster_test.go | 15 ++++++--------- metakube/resource_maintenance_cronjob_test.go | 2 +- metakube/resource_node_deployment_test.go | 5 ++--- metakube/resource_role_binding_test.go | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/metakube/resource_cluster_role_binding_test.go b/metakube/resource_cluster_role_binding_test.go index 531872a..09fd42e 100644 --- a/metakube/resource_cluster_role_binding_test.go +++ b/metakube/resource_cluster_role_binding_test.go @@ -15,7 +15,7 @@ func TestAccMetakubeClusterRoleBinding(t *testing.T) { t.Parallel() resourceName := "metakube_cluster_role_binding.acctest" params := &testAccCheckMetaKubeClusterRoleBindingBasicParams{ - ClusterName: randomName("testacc", 5), + ClusterName: makeRandomName() + "-cluster-role-binding", DatacenterName: os.Getenv(testEnvOpenstackNodeDC), ProjectID: os.Getenv(testEnvProjectID), Version: os.Getenv(testEnvK8sVersionOpenstack), diff --git a/metakube/resource_cluster_test.go b/metakube/resource_cluster_test.go index 08621c9..b397c5c 100644 --- a/metakube/resource_cluster_test.go +++ b/metakube/resource_cluster_test.go @@ -372,9 +372,8 @@ resource "metakube_cluster" "acctest_cluster" { cloud { openstack { application_credentials { - project_id = "{{ .OpenstackProjectID }}" - application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" - application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" + id = "{{ .OpenstackApplicationCredentialsID }}" + secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" security_group = openstack_networking_secgroup_v2.cluster-net.name @@ -614,9 +613,8 @@ resource "metakube_cluster" "acctest_cluster" { cloud { openstack { application_credentials { - project_id = "{{ .OpenstackProjectID }}" - application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" - application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" + id = "{{ .OpenstackApplicationCredentialsID }}" + secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" } @@ -646,9 +644,8 @@ resource "metakube_cluster" "acctest_cluster" { cloud { openstack { application_credentials { - project_id = "{{ .OpenstackProjectID }}" - application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" - application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" + id = "{{ .OpenstackApplicationCredentialsID }}" + secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" } diff --git a/metakube/resource_maintenance_cronjob_test.go b/metakube/resource_maintenance_cronjob_test.go index b39560c..e05d41c 100644 --- a/metakube/resource_maintenance_cronjob_test.go +++ b/metakube/resource_maintenance_cronjob_test.go @@ -19,7 +19,7 @@ func TestAccMetakubeCluster_MaintenanceCronJob_Basic(t *testing.T) { resourceName := "metakube_maintenance_cron_job.acctest" params := &testAccCheckMetaKubeMaintenanceCronJobBasicParams{ - ClusterName: randomName("testacc", 5), + ClusterName: makeRandomName() + "-maint-cron-job", DatacenterName: os.Getenv(testEnvOpenstackNodeDC), ProjectID: os.Getenv(testEnvProjectID), Version: os.Getenv(testEnvK8sVersionOpenstack), diff --git a/metakube/resource_node_deployment_test.go b/metakube/resource_node_deployment_test.go index bf8aa8e..a267cc2 100644 --- a/metakube/resource_node_deployment_test.go +++ b/metakube/resource_node_deployment_test.go @@ -189,9 +189,8 @@ var nodeDeploymentBasicTemplate = mustParseTemplate("nodeDeploymentBasic", ` cloud { openstack { application_credentials { - project_id = "{{ .OpenstackProjectID }}" - application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" - application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" + id = "{{ .OpenstackApplicationCredentialsID }}" + secret = "{{ .OpenstackApplicationCredentialsSecret }}" } floating_ip_pool = "ext-net" } diff --git a/metakube/resource_role_binding_test.go b/metakube/resource_role_binding_test.go index 38176e1..f46e518 100644 --- a/metakube/resource_role_binding_test.go +++ b/metakube/resource_role_binding_test.go @@ -15,7 +15,7 @@ func TestAccMetakubeRoleBinding(t *testing.T) { t.Parallel() resourceName := "metakube_role_binding.acctest" params := &testAccCheckMetaKubeRoleBindingBasicParams{ - ClusterName: randomName("testacc", 5), + ClusterName: makeRandomName() + "-role-binding", DatacenterName: os.Getenv(testEnvOpenstackNodeDC), ProjectID: os.Getenv(testEnvProjectID), Version: os.Getenv(testEnvK8sVersionOpenstack), From d30b7d276d425c289c02deca3a93dd033f3a6c7d Mon Sep 17 00:00:00 2001 From: Robert Nemeti Date: Thu, 4 Jul 2024 23:25:46 +0200 Subject: [PATCH 4/4] use os user credentials for dynamic app credentials --- metakube/provider_test.go | 2 ++ metakube/resource_cluster.go | 31 +++++++++++++---- metakube/resource_cluster_test.go | 55 +++++++++++++++++-------------- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/metakube/provider_test.go b/metakube/provider_test.go index 82631c3..e3299f7 100644 --- a/metakube/provider_test.go +++ b/metakube/provider_test.go @@ -27,7 +27,9 @@ const ( testEnvOpenstackNodeDC = "METAKUBE_OPENSTACK_NODE_DC" testEnvOpenstackApplicationCredentialsID = "METAKUBE_OPENSTACK_APPLICATION_CREDENTIALS_ID" testEnvOpenstackApplicationCredentialsSecret = "METAKUBE_OPENSTACK_APPLICATION_CREDENTIALS_SECRET" + testEnvOpenstackUsername = "METAKUBE_OPENSTACK_USERNAME" testEnvOpenstackAuthURL = "METAKUBE_OPENSTACK_AUTH_URL" + testEnvOpenstackPassword = "METAKUBE_OPENSTACK_PASSWORD" testEnvOpenstackProjectID = "METAKUBE_OPENSTACK_PROJECT_ID" testEnvOpenstackProjectName = "METAKUBE_OPENSTACK_PROJECT_NAME" testEnvOpenstackRegion = "METAKUBE_OPENSTACK_REGION" diff --git a/metakube/resource_cluster.go b/metakube/resource_cluster.go index 08cc6d5..f9551cc 100644 --- a/metakube/resource_cluster.go +++ b/metakube/resource_cluster.go @@ -748,18 +748,37 @@ func metakubeResourceClusterWaitForReady(ctx context.Context, k *metakubeProvide p.SetProjectID(projectID) p.SetClusterID(clusterID) - r, err := k.client.Project.GetClusterV2(p, k.auth) + cluster, err := k.client.Project.GetClusterV2(p, k.auth) if err != nil { return retry.RetryableError(fmt.Errorf("unable to get cluster '%s': %s", clusterID, stringifyResponseError(err))) } - if configuredVersion == "" { - return nil - } else if r.Payload.Status.Version == models.Semver(configuredVersion) { - return nil + p1 := project.NewGetClusterHealthV2Params() + p1.SetContext(ctx) + p1.SetProjectID(projectID) + p1.SetClusterID(clusterID) + + clusterHealth, err := k.client.Project.GetClusterHealthV2(p1, k.auth) + if err != nil { + return retry.RetryableError(fmt.Errorf("unable to get cluster '%s' health: %s", clusterID, stringifyResponseError(err))) + } + + const up models.HealthStatus = 1 + if clusterHealth.Payload.Apiserver == up && + clusterHealth.Payload.CloudProviderInfrastructure == up && + clusterHealth.Payload.Controller == up && + clusterHealth.Payload.Etcd == up && + clusterHealth.Payload.MachineController == up && + clusterHealth.Payload.Scheduler == up && + clusterHealth.Payload.UserClusterControllerManager == up { + if configuredVersion == "" { + return nil + } else if cluster.Payload.Status.Version == models.Semver(configuredVersion) { + return nil + } } - k.log.Debugf("waiting for cluster '%s' to be ready, %+v", clusterID, r.Payload) + k.log.Debugf("waiting for cluster '%s' to be ready, %+v", clusterID, clusterHealth.Payload) return retry.RetryableError(fmt.Errorf("waiting for cluster '%s' to be ready", clusterID)) }) } diff --git a/metakube/resource_cluster_test.go b/metakube/resource_cluster_test.go index b397c5c..c74556d 100644 --- a/metakube/resource_cluster_test.go +++ b/metakube/resource_cluster_test.go @@ -51,7 +51,7 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) { resourceName := "metakube_cluster.acctest_cluster" data := &clusterOpenstackBasicData{ - Name: makeRandomName() + "-basic", + Name: makeRandomName() + "-cluster-os-basic", OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), @@ -159,7 +159,7 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"spec.0.cloud.0.openstack.0.user_credentials", "kube_login_kube_config", "oidc_kube_config"}, + ImportStateVerifyIgnore: []string{"spec.0.cloud.0.openstack.0.application_credentials", "kube_login_kube_config", "oidc_kube_config"}, }, { Config: config2.String(), @@ -171,7 +171,7 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) { ImportState: true, ImportStateVerify: false, ImportStateId: "123abc", - ExpectError: regexp.MustCompile(`(Please verify the ID is correct|Cannot import non-existent remote object)`), + ExpectError: regexp.MustCompile(`(no object exists with the given id|Cannot import non-existent remote object)`), }, }, }) @@ -222,25 +222,29 @@ func TestAccMetakubeCluster_Openstack_ApplicationCredentials_Dynammic(t *testing var cluster models.Cluster resourceName := "metakube_cluster.acctest_cluster" data := &clusterOpenstackApplicationCredentailsData{ - Name: makeRandomName() + "-appcred-dynamic", - OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), - OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), - OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), - OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), - OpenstackRegion: os.Getenv(testEnvOpenstackRegion), - DatacenterName: os.Getenv(testEnvOpenstackNodeDC), - ProjectID: os.Getenv(testEnvProjectID), - Version: os.Getenv(testEnvK8sVersionOpenstack), - OpenstackApplicationCredentialID: os.Getenv(testEnvOpenstackApplicationCredentialsID), - OpenstackApplicationCredentialSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), - Dynamic: true, + Name: makeRandomName() + "-appcred-dynamic", + OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), + OpenstackUser: os.Getenv(testEnvOpenstackUsername), + OpenstackPassword: os.Getenv(testEnvOpenstackPassword), + OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID), + OpenstackRegion: os.Getenv(testEnvOpenstackRegion), + DatacenterName: os.Getenv(testEnvOpenstackNodeDC), + ProjectID: os.Getenv(testEnvProjectID), + Version: os.Getenv(testEnvK8sVersionOpenstack), + OpenstackApplicationCredentialID: os.Getenv(testEnvOpenstackApplicationCredentialsID), + OpenstackApplicationCredentialSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret), + Dynamic: true, } var config strings.Builder if err := clusterOpenstackApplicationCredentialsBasicTemplate.Execute(&config, data); err != nil { t.Fatal(err) } resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheckForOpenstack(t) }, + PreCheck: func() { + testAccPreCheckForOpenstack(t) + checkEnv(t, testEnvOpenstackUsername) + checkEnv(t, testEnvOpenstackPassword) + }, Providers: testAccProviders, ExternalProviders: map[string]resource.ExternalProvider{ "openstack": { @@ -267,7 +271,7 @@ func TestAccMetakubeCluster_Openstack_UpgradeVersion(t *testing.T) { resourceName := "metakube_cluster.acctest_cluster" versionedConfig := func(version string) string { data := &clusterOpenstackBasicData{ - Name: makeRandomName() + "-upgrade", + Name: makeRandomName() + "-cluster-os-upgrade", Version: version, OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL), OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID), @@ -424,11 +428,11 @@ resource "openstack_networking_subnet_v2" "subnet_tf_test" { }`) type clusterOpenstackApplicationCredentailsData struct { - OpenstackAuthURL string - OpenstackApplicationCredentialsID string - OpenstackApplicationCredentialsSecret string - OpenstackProjectID string - OpenstackRegion string + OpenstackAuthURL string + OpenstackUser string + OpenstackPassword string + OpenstackProjectID string + OpenstackRegion string Name string DatacenterName string @@ -451,8 +455,9 @@ terraform { {{ if .Dynamic }} provider "openstack" { auth_url = "{{ .OpenstackAuthURL }}" - application_credential_id = "{{ .OpenstackApplicationCredentialsID }}" - application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}" + user_name = "{{ .OpenstackUser }}" + password = "{{ .OpenstackPassword }}" + tenant_id = "{{ .OpenstackProjectID }}" region = "{{ .OpenstackRegion }}" } @@ -696,7 +701,7 @@ func TestAccMetakubeCluster_AWS_Basic(t *testing.T) { var cluster models.Cluster resourceName := "metakube_cluster.acctest_cluster" data := &clusterAWSBasicData{ - Name: makeRandomName() + "-aws-basic", + Name: makeRandomName() + "-cluster-aws-basic", ProjectID: os.Getenv(testEnvProjectID), AccessID: os.Getenv(testEnvAWSAccessKeyID), AccessSecret: os.Getenv(testAWSSecretAccessKey),