Skip to content

Commit

Permalink
Merge pull request #106 from syseleven/K8S-10217_fix_cluster_update_wait
Browse files Browse the repository at this point in the history 
K8s-10217 fix cluster update wait
  • Loading branch information
@nrobert13
nrobert13 authored Jul 10, 2024
2 parents e9d599b + d30b7d2 commit bd1f1fe
Show file tree
Hide file tree
Showing 9 changed files with 118 additions and 103 deletions.
4 changes: 2 additions & 2 deletions metakube/provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ func testAccPreCheckForOpenstack(t *testing.T) {
t.Helper()
testAccPreCheck(t)
checkEnv(t, testEnvK8sVersionOpenstack)
checkEnv(t, testEnvOpenstackUsername)
checkEnv(t, testEnvOpenstackPassword)
checkEnv(t, testEnvOpenstackApplicationCredentialsID)
checkEnv(t, testEnvOpenstackApplicationCredentialsSecret)
checkEnv(t, testEnvOpenstackProjectID)
checkEnv(t, testEnvOpenstackProjectName)
checkEnv(t, testEnvOpenstackRegion)
Expand Down
47 changes: 32 additions & 15 deletions metakube/resource_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ func metakubeResourceClusterCreate(ctx context.Context, d *schema.ResourceData,
return diag.FromErr(err)
}

if err := metakubeResourceClusterWaitForReady(ctx, meta, d.Timeout(schema.TimeoutCreate), projectID, d.Id()); err != nil {
if err := metakubeResourceClusterWaitForReady(ctx, meta, d.Timeout(schema.TimeoutCreate), projectID, d.Id(), ""); err != nil {
// In case of timeout, we still want to return the cluster resource
retDiags = append(retDiags, metakubeResourceClusterRead(ctx, d, m)...)
retDiags = append(retDiags, diag.Diagnostic{
Expand Down Expand Up @@ -610,7 +610,11 @@ func metakubeResourceClusterUpdate(ctx context.Context, d *schema.ResourceData,
}
}

if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutUpdate), projectID, d.Id()); err != nil {
configuredVersion, ok := d.GetOk("spec.0.version")
if !ok {
return nil
}
if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutUpdate), projectID, d.Id(), configuredVersion.(string)); err != nil {
return diag.Errorf("cluster '%s' is not ready: %v", d.Id(), err)
}

Expand Down Expand Up @@ -736,32 +740,45 @@ func assignSSHKeysToCluster(projectID, clusterID string, sshkeyIDs []string, k *
return nil
}

func metakubeResourceClusterWaitForReady(ctx context.Context, k *metakubeProviderMeta, timeout time.Duration, projectID, clusterID string) error {
func metakubeResourceClusterWaitForReady(ctx context.Context, k *metakubeProviderMeta, timeout time.Duration, projectID, clusterID, configuredVersion string) error {
return retry.RetryContext(ctx, timeout, func() *retry.RetryError {

p := project.NewGetClusterHealthV2Params()
p := project.NewGetClusterV2Params()
p.SetContext(ctx)
p.SetProjectID(projectID)
p.SetClusterID(clusterID)

r, err := k.client.Project.GetClusterHealthV2(p, k.auth)
cluster, err := k.client.Project.GetClusterV2(p, k.auth)
if err != nil {
return retry.RetryableError(fmt.Errorf("unable to get cluster '%s': %s", clusterID, stringifyResponseError(err)))
}

p1 := project.NewGetClusterHealthV2Params()
p1.SetContext(ctx)
p1.SetProjectID(projectID)
p1.SetClusterID(clusterID)

clusterHealth, err := k.client.Project.GetClusterHealthV2(p1, k.auth)
if err != nil {
return retry.RetryableError(fmt.Errorf("unable to get cluster '%s' health: %s", clusterID, stringifyResponseError(err)))
}

const up models.HealthStatus = 1

if r.Payload.Apiserver == up &&
r.Payload.CloudProviderInfrastructure == up &&
r.Payload.Controller == up &&
r.Payload.Etcd == up &&
r.Payload.MachineController == up &&
r.Payload.Scheduler == up &&
r.Payload.UserClusterControllerManager == up {
return nil
if clusterHealth.Payload.Apiserver == up &&
clusterHealth.Payload.CloudProviderInfrastructure == up &&
clusterHealth.Payload.Controller == up &&
clusterHealth.Payload.Etcd == up &&
clusterHealth.Payload.MachineController == up &&
clusterHealth.Payload.Scheduler == up &&
clusterHealth.Payload.UserClusterControllerManager == up {
if configuredVersion == "" {
return nil
} else if cluster.Payload.Status.Version == models.Semver(configuredVersion) {
return nil
}
}

k.log.Debugf("waiting for cluster '%s' to be ready, %+v", clusterID, r.Payload)
k.log.Debugf("waiting for cluster '%s' to be ready, %+v", clusterID, clusterHealth.Payload)
return retry.RetryableError(fmt.Errorf("waiting for cluster '%s' to be ready", clusterID))
})
}
Expand Down
2 changes: 1 addition & 1 deletion metakube/resource_cluster_role_binding_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestAccMetakubeClusterRoleBinding(t *testing.T) {
t.Parallel()
resourceName := "metakube_cluster_role_binding.acctest"
params := &testAccCheckMetaKubeClusterRoleBindingBasicParams{
ClusterName: randomName("testacc", 5),
ClusterName: makeRandomName() + "-cluster-role-binding",
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
Version: os.Getenv(testEnvK8sVersionOpenstack),
Expand Down
112 changes: 56 additions & 56 deletions metakube/resource_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,15 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) {

resourceName := "metakube_cluster.acctest_cluster"
data := &clusterOpenstackBasicData{
Name: makeRandomName() + "-basic",
OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL),
OpenstackUser: os.Getenv(testEnvOpenstackUsername),
OpenstackPassword: os.Getenv(testEnvOpenstackPassword),
OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID),
OpenstackRegion: os.Getenv(testEnvOpenstackRegion),
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
Version: os.Getenv(testEnvK8sVersionOpenstack),
Name: makeRandomName() + "-cluster-os-basic",
OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL),
OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID),
OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret),
OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID),
OpenstackRegion: os.Getenv(testEnvOpenstackRegion),
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
Version: os.Getenv(testEnvK8sVersionOpenstack),
}
var config strings.Builder
if err := clusterOpenstackBasicTemplate.Execute(&config, data); err != nil {
Expand Down Expand Up @@ -159,7 +159,7 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) {
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"spec.0.cloud.0.openstack.0.user_credentials", "kube_login_kube_config", "oidc_kube_config"},
ImportStateVerifyIgnore: []string{"spec.0.cloud.0.openstack.0.application_credentials", "kube_login_kube_config", "oidc_kube_config"},
},
{
Config: config2.String(),
Expand All @@ -171,7 +171,7 @@ func TestAccMetakubeCluster_Openstack_Basic(t *testing.T) {
ImportState: true,
ImportStateVerify: false,
ImportStateId: "123abc",
ExpectError: regexp.MustCompile(`(Please verify the ID is correct|Cannot import non-existent remote object)`),
ExpectError: regexp.MustCompile(`(no object exists with the given id|Cannot import non-existent remote object)`),
},
},
})
Expand Down Expand Up @@ -240,7 +240,11 @@ func TestAccMetakubeCluster_Openstack_ApplicationCredentials_Dynammic(t *testing
t.Fatal(err)
}
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheckForOpenstack(t) },
PreCheck: func() {
testAccPreCheckForOpenstack(t)
checkEnv(t, testEnvOpenstackUsername)
checkEnv(t, testEnvOpenstackPassword)
},
Providers: testAccProviders,
ExternalProviders: map[string]resource.ExternalProvider{
"openstack": {
Expand All @@ -267,15 +271,15 @@ func TestAccMetakubeCluster_Openstack_UpgradeVersion(t *testing.T) {
resourceName := "metakube_cluster.acctest_cluster"
versionedConfig := func(version string) string {
data := &clusterOpenstackBasicData{
Name: makeRandomName() + "-upgrade",
Version: version,
OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL),
OpenstackUser: os.Getenv(testEnvOpenstackUsername),
OpenstackPassword: os.Getenv(testEnvOpenstackPassword),
OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID),
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
OpenstackRegion: os.Getenv(testEnvOpenstackRegion),
Name: makeRandomName() + "-cluster-os-upgrade",
Version: version,
OpenstackAuthURL: os.Getenv(testEnvOpenstackAuthURL),
OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID),
OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret),
OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID),
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
OpenstackRegion: os.Getenv(testEnvOpenstackRegion),
}
var result strings.Builder
if err := clusterOpenstackBasicTemplate.Execute(&result, data); err != nil {
Expand Down Expand Up @@ -315,11 +319,11 @@ func TestAccMetakubeCluster_Openstack_UpgradeVersion(t *testing.T) {
}

type clusterOpenstackBasicData struct {
OpenstackAuthURL string
OpenstackUser string
OpenstackPassword string
OpenstackProjectID string
OpenstackRegion string
OpenstackAuthURL string
OpenstackApplicationCredentialsID string
OpenstackApplicationCredentialsSecret string
OpenstackProjectID string
OpenstackRegion string

Name string
DatacenterName string
Expand All @@ -342,9 +346,8 @@ terraform {
provider "openstack" {
auth_url = "{{ .OpenstackAuthURL }}"
user_name = "{{ .OpenstackUser }}"
password = "{{ .OpenstackPassword }}"
tenant_id = "{{ .OpenstackProjectID }}"
application_credential_id = "{{ .OpenstackApplicationCredentialsID }}"
application_credential_secret = "{{ .OpenstackApplicationCredentialsSecret }}"
region = "{{ .OpenstackRegion }}"
}
Expand Down Expand Up @@ -372,10 +375,9 @@ resource "metakube_cluster" "acctest_cluster" {
}
cloud {
openstack {
user_credentials {
project_id = "{{ .OpenstackProjectID }}"
username = "{{ .OpenstackUser }}"
password = "{{ .OpenstackPassword }}"
application_credentials {
id = "{{ .OpenstackApplicationCredentialsID }}"
secret = "{{ .OpenstackApplicationCredentialsSecret }}"
}
floating_ip_pool = "ext-net"
security_group = openstack_networking_secgroup_v2.cluster-net.name
Expand Down Expand Up @@ -544,13 +546,13 @@ func TestAccMetakubeCluster_SSHKeys(t *testing.T) {
resourceName := "metakube_cluster.acctest_cluster"

data := &clusterOpenstackWithSSHKeyData{
Name: makeRandomName() + "-sshkeys",
OpenstackUser: os.Getenv(testEnvOpenstackUsername),
OpenstackPassword: os.Getenv(testEnvOpenstackPassword),
OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID),
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
Version: os.Getenv(testEnvK8sVersionOpenstack),
Name: makeRandomName() + "-sshkeys",
OpenstackApplicationCredentialsID: os.Getenv(testEnvOpenstackApplicationCredentialsID),
OpenstackApplicationCredentialsSecret: os.Getenv(testEnvOpenstackApplicationCredentialsSecret),
OpenstackProjectID: os.Getenv(testEnvOpenstackProjectID),
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
Version: os.Getenv(testEnvK8sVersionOpenstack),
}

var config1 strings.Builder
Expand Down Expand Up @@ -591,13 +593,13 @@ func TestAccMetakubeCluster_SSHKeys(t *testing.T) {
}

type clusterOpenstackWithSSHKeyData struct {
Name string
DatacenterName string
ProjectID string
Version string
OpenstackProjectID string
OpenstackUser string
OpenstackPassword string
Name string
DatacenterName string
ProjectID string
Version string
OpenstackProjectID string
OpenstackApplicationCredentialsID string
OpenstackApplicationCredentialsSecret string
}

var clusterOpenstackTemplateWithSSHKey1 = mustParseTemplate("clusterOpenstackWithSSHKey1", `
Expand All @@ -615,10 +617,9 @@ resource "metakube_cluster" "acctest_cluster" {
enable_ssh_agent = true
cloud {
openstack {
user_credentials {
project_id = "{{ .OpenstackProjectID }}"
username = "{{ .OpenstackUser }}"
password = "{{ .OpenstackPassword }}"
application_credentials {
id = "{{ .OpenstackApplicationCredentialsID }}"
secret = "{{ .OpenstackApplicationCredentialsSecret }}"
}
floating_ip_pool = "ext-net"
}
Expand Down Expand Up @@ -647,10 +648,9 @@ resource "metakube_cluster" "acctest_cluster" {
enable_ssh_agent = true
cloud {
openstack {
user_credentials {
project_id = "{{ .OpenstackProjectID }}"
username = "{{ .OpenstackUser }}"
password = "{{ .OpenstackPassword }}"
application_credentials {
id = "{{ .OpenstackApplicationCredentialsID }}"
secret = "{{ .OpenstackApplicationCredentialsSecret }}"
}
floating_ip_pool = "ext-net"
}
Expand Down Expand Up @@ -701,7 +701,7 @@ func TestAccMetakubeCluster_AWS_Basic(t *testing.T) {
var cluster models.Cluster
resourceName := "metakube_cluster.acctest_cluster"
data := &clusterAWSBasicData{
Name: makeRandomName() + "-aws-basic",
Name: makeRandomName() + "-cluster-aws-basic",
ProjectID: os.Getenv(testEnvProjectID),
AccessID: os.Getenv(testEnvAWSAccessKeyID),
AccessSecret: os.Getenv(testAWSSecretAccessKey),
Expand Down
2 changes: 1 addition & 1 deletion metakube/resource_maintenance_cronjob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func metakubeResourceMaintenanceCronJobCreate(ctx context.Context, d *schema.Res
}

// TODO not sure to remove this
if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID); err != nil {
if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID, ""); err != nil {
return diag.Errorf("cluster is not ready: %v", err)
}

Expand Down
2 changes: 1 addition & 1 deletion metakube/resource_maintenance_cronjob_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func TestAccMetakubeCluster_MaintenanceCronJob_Basic(t *testing.T) {

resourceName := "metakube_maintenance_cron_job.acctest"
params := &testAccCheckMetaKubeMaintenanceCronJobBasicParams{
ClusterName: randomName("testacc", 5),
ClusterName: makeRandomName() + "-maint-cron-job",
DatacenterName: os.Getenv(testEnvOpenstackNodeDC),
ProjectID: os.Getenv(testEnvProjectID),
Version: os.Getenv(testEnvK8sVersionOpenstack),
Expand Down
2 changes: 1 addition & 1 deletion metakube/resource_node_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func metakubeResourceNodeDeploymentCreate(ctx context.Context, d *schema.Resourc
return diag.FromErr(err)
}

if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID); err != nil {
if err := metakubeResourceClusterWaitForReady(ctx, k, d.Timeout(schema.TimeoutCreate), projectID, clusterID, ""); err != nil {
return diag.Errorf("cluster is not ready: %v", err)
}

Expand Down
Loading

0 comments on commit bd1f1fe

Please sign in to comment.