From 58bc1989828cfdc8c8d02fd9deb0bfdc440b5c55 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 14 Nov 2024 22:28:26 +0100 Subject: [PATCH] Refine v1beta2 Available condition in KCP --- .../kubeadm/internal/controllers/status.go | 108 ++++++++++- .../internal/controllers/status_test.go | 171 ++++++++++++++++-- 2 files changed, 255 insertions(+), 24 deletions(-) diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index 3ee2813993de..57d20f904dbd 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -474,6 +474,20 @@ func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControl if etcdIsManaged { if etcdMembers == nil { + // In case the control plane just initialized, give some more time before reporting failed to get etcd members. + // Note: Two minutes is the time after which we assume that not getting the list of etcd members is an actual problem. + if c := v1beta2conditions.Get(kcp, controlplanev1.KubeadmControlPlaneInitializedV1Beta2Condition); c != nil && + c.Status == metav1.ConditionTrue && + time.Since(c.LastTransitionTime.Time) < 2*time.Minute { + v1beta2conditions.Set(kcp, metav1.Condition{ + Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, + Message: "Waiting for etcd to report the list of members", + }) + return + } + v1beta2conditions.Set(kcp, metav1.Condition{ Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionUnknown, @@ -520,7 +534,9 @@ func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControl // etcd members might not match with machines, e.g. while provisioning a new machine. etcdQuorum := (len(etcdMembers) / 2.0) + 1 k8sControlPlaneHealthy := 0 + k8sControlPlaneNotHealthy := 0 etcdMembersHealthy := 0 + etcdMembersNotHealthy := 0 for _, machine := range machines { // if external etcd, only look at the status of the K8s control plane components on this machine. if !etcdIsManaged { @@ -528,6 +544,11 @@ func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControl v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition) && v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition) { k8sControlPlaneHealthy++ + } else if shouldSurfaceWhenAvailableTrue(machine, + controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, + controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition, + controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition) { + k8sControlPlaneNotHealthy++ } continue } @@ -541,6 +562,9 @@ func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControl if v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition) { etcdMembersHealthy++ + } else if shouldSurfaceWhenAvailableTrue(machine, + controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition) { + etcdMembersNotHealthy++ } if v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition) && @@ -549,6 +573,13 @@ func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControl v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition) && v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyV1Beta2Condition) { k8sControlPlaneHealthy++ + } else if shouldSurfaceWhenAvailableTrue(machine, + controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, + controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition, + controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition, + controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, + controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyV1Beta2Condition) { + k8sControlPlaneNotHealthy++ } } @@ -556,46 +587,103 @@ func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControl (!etcdIsManaged || etcdMembersHealthy >= etcdQuorum) && k8sControlPlaneHealthy >= 1 && v1beta2conditions.IsTrue(kcp, controlplanev1.KubeadmControlPlaneCertificatesAvailableV1Beta2Condition) { + messages := []string{} + + if etcdIsManaged && etcdMembersNotHealthy > 0 { + switch len(etcdMembers) - etcdMembersNotHealthy { + case 1: + messages = append(messages, fmt.Sprintf("* 1 of %d etcd members is healthy, at least %d required for etcd quorum", len(etcdMembers), etcdQuorum)) + default: + messages = append(messages, fmt.Sprintf("* %d of %d etcd members are healthy, at least %d required for etcd quorum", len(etcdMembers)-etcdMembersNotHealthy, len(etcdMembers), etcdQuorum)) + } + } + + if k8sControlPlaneNotHealthy > 0 { + switch len(machines) - k8sControlPlaneNotHealthy { + case 1: + messages = append(messages, fmt.Sprintf("* 1 of %d Machines has healthy control plane components, at least 1 required", len(machines))) + default: + messages = append(messages, fmt.Sprintf("* %d of %d Machines have healthy control plane components, at least 1 required", len(machines)-k8sControlPlaneNotHealthy, len(machines))) + } + } + v1beta2conditions.Set(kcp, metav1.Condition{ - Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, - Status: metav1.ConditionTrue, - Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason, + Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, + Status: metav1.ConditionTrue, + Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason, + Message: strings.Join(messages, "\n"), }) return } messages := []string{} if !kcp.DeletionTimestamp.IsZero() { - messages = append(messages, "Control plane metadata.deletionTimestamp is set") + messages = append(messages, "* Control plane metadata.deletionTimestamp is set") } if !v1beta2conditions.IsTrue(kcp, controlplanev1.KubeadmControlPlaneCertificatesAvailableV1Beta2Condition) { - messages = append(messages, "Control plane certificates are not available") + messages = append(messages, "* Control plane certificates are not available") } if etcdIsManaged && etcdMembersHealthy < etcdQuorum { switch etcdMembersHealthy { case 0: - messages = append(messages, fmt.Sprintf("There are no healthy etcd member, at least %d required for etcd quorum", etcdQuorum)) + messages = append(messages, fmt.Sprintf("* There are no healthy etcd member, at least %d required for etcd quorum", etcdQuorum)) case 1: - messages = append(messages, fmt.Sprintf("There is 1 healthy etcd member, at least %d required for etcd quorum", etcdQuorum)) + messages = append(messages, fmt.Sprintf("* 1 of %d etcd members is healthy, at least %d required for etcd quorum", len(etcdMembers), etcdQuorum)) default: - messages = append(messages, fmt.Sprintf("There are %d healthy etcd members, at least %d required for etcd quorum", etcdMembersHealthy, etcdQuorum)) + messages = append(messages, fmt.Sprintf("* %d of %d etcd members are healthy, at least %d required for etcd quorum", etcdMembersHealthy, len(etcdMembers), etcdQuorum)) } } if k8sControlPlaneHealthy < 1 { - messages = append(messages, "There are no Machines with healthy control plane components, at least 1 required") + messages = append(messages, "* There are no Machines with healthy control plane components, at least 1 required") } v1beta2conditions.Set(kcp, metav1.Condition{ Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionFalse, Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, - Message: strings.Join(messages, ";"), + Message: strings.Join(messages, "\n"), }) } +// shouldSurfaceWhenAvailableTrue defines when a control plane components/etcd issue should surface when +// Available condition is true. +// The main goal of this check is to avoid to surface false negatives/flakes, and thus it requires that +// an issue exists for at least more than 10 seconds before surfacing it. +func shouldSurfaceWhenAvailableTrue(machine *clusterv1.Machine, conditionTypes ...string) bool { + // Get the min time when one of the conditions in input transitioned to false or unknown. + var t *time.Time + for _, conditionType := range conditionTypes { + c := v1beta2conditions.Get(machine, conditionType) + if c == nil { + continue + } + if c.Status == metav1.ConditionTrue { + continue + } + if t == nil { + t = ptr.To(c.LastTransitionTime.Time) + } + t = ptr.To(minTime(*t, c.LastTransitionTime.Time)) + } + + if t != nil { + if time.Since(*t) > 10*time.Second { + return true + } + } + return false +} + +func minTime(t1, t2 time.Time) time.Time { + if t1.After(t2) { + return t2 + } + return t1 +} + func aggregateStaleMachines(machines collections.Machines) string { if len(machines) == 0 { return "" diff --git a/controlplane/kubeadm/internal/controllers/status_test.go b/controlplane/kubeadm/internal/controllers/status_test.go index 78c73059d22b..08142d1a51b8 100644 --- a/controlplane/kubeadm/internal/controllers/status_test.go +++ b/controlplane/kubeadm/internal/controllers/status_test.go @@ -658,18 +658,62 @@ func TestDeletingCondition(t *testing.T) { } } +func Test_shouldSurfaceWhenAvailableTrue(t *testing.T) { + reconcileTime := time.Now() + + apiServerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} + apiServerPodNotHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionFalse, LastTransitionTime: metav1.Time{Time: reconcileTime}} + apiServerPodNotHealthy11s := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionFalse, LastTransitionTime: metav1.Time{Time: reconcileTime.Add(-11 * time.Second)}} + + etcdMemberHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} + etcdMemberNotHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, Status: metav1.ConditionFalse, LastTransitionTime: metav1.Time{Time: reconcileTime}} + + testCases := []struct { + name string + machine *clusterv1.Machine + want bool + }{ + { + name: "Machine doesn't have issues, it should not surface", + machine: &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodHealthy, etcdMemberHealthy}}}}, + want: false, + }, + { + name: "Machine has issue set by less than 10s it should not surface", + machine: &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodNotHealthy, etcdMemberNotHealthy}}}}, + want: false, + }, + { + name: "Machine has at least one issue set by more than 10s it should surface", + machine: &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodNotHealthy11s, etcdMemberNotHealthy}}}}, + want: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + got := shouldSurfaceWhenAvailableTrue(tc.machine, controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition) + g.Expect(got).To(Equal(tc.want)) + }) + } +} + func Test_setAvailableCondition(t *testing.T) { + reconcileTime := time.Now() + certificatesReady := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneCertificatesAvailableV1Beta2Condition, Status: metav1.ConditionTrue} certificatesNotReady := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneCertificatesAvailableV1Beta2Condition, Status: metav1.ConditionFalse} - apiServerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue} - apiServerPodNotHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionFalse} - controllerManagerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue} - schedulerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue} - etcdPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue} + apiServerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} + apiServerPodNotHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionFalse, LastTransitionTime: metav1.Time{Time: reconcileTime}} + apiServerPodNotHealthy11s := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition, Status: metav1.ConditionFalse, LastTransitionTime: metav1.Time{Time: reconcileTime.Add(-11 * time.Second)}} + controllerManagerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} + schedulerPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} + etcdPodHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} - etcdMemberHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, Status: metav1.ConditionTrue} - etcdMemberNotHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, Status: metav1.ConditionFalse} + etcdMemberHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, Status: metav1.ConditionTrue, LastTransitionTime: metav1.Time{Time: reconcileTime}} + etcdMemberNotHealthy := metav1.Condition{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition, Status: metav1.ConditionFalse, LastTransitionTime: metav1.Time{Time: reconcileTime}} tests := []struct { name string @@ -699,7 +743,7 @@ func Test_setAvailableCondition(t *testing.T) { }, }, { - name: "Failed to get etcd members", + name: "Failed to get etcd members right after being initialized", controlPlane: &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{ Spec: controlplanev1.KubeadmControlPlaneSpec{ @@ -709,7 +753,43 @@ func Test_setAvailableCondition(t *testing.T) { }, }, }, - Status: controlplanev1.KubeadmControlPlaneStatus{Initialized: true}, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + V1Beta2: &controlplanev1.KubeadmControlPlaneV1Beta2Status{ + Conditions: []metav1.Condition{ + {Type: controlplanev1.KubeadmControlPlaneInitializedV1Beta2Condition, Status: metav1.ConditionTrue, Reason: controlplanev1.KubeadmControlPlaneInitializedV1Beta2Reason, LastTransitionTime: metav1.Time{Time: reconcileTime.Add(-5 * time.Second)}}, + }, + }, + }, + }, + EtcdMembers: nil, + }, + expectCondition: metav1.Condition{ + Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, + Message: "Waiting for etcd to report the list of members", + }, + }, + { + name: "Failed to get etcd members, 2m after the cluster was initialized", + controlPlane: &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + Etcd: bootstrapv1.Etcd{Local: &bootstrapv1.LocalEtcd{}}, + }, + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + V1Beta2: &controlplanev1.KubeadmControlPlaneV1Beta2Status{ + Conditions: []metav1.Condition{ + {Type: controlplanev1.KubeadmControlPlaneInitializedV1Beta2Condition, Status: metav1.ConditionTrue, Reason: controlplanev1.KubeadmControlPlaneInitializedV1Beta2Reason, LastTransitionTime: metav1.Time{Time: reconcileTime.Add(-5 * time.Minute)}}, + }, + }, + }, }, EtcdMembers: nil, }, @@ -817,6 +897,34 @@ func Test_setAvailableCondition(t *testing.T) { Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason, }, }, + { + name: "KCP is available, some control plane failures to be reported", + controlPlane: &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + V1Beta2: &controlplanev1.KubeadmControlPlaneV1Beta2Status{ + Conditions: []metav1.Condition{certificatesReady}, + }, + }, + }, + Machines: collections.FromMachines( + &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodHealthy, controllerManagerPodHealthy, schedulerPodHealthy, etcdPodHealthy, etcdMemberHealthy}}}}, + &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m2"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodNotHealthy, controllerManagerPodHealthy, schedulerPodHealthy, etcdPodHealthy, etcdMemberHealthy}}}}, + &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m3"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodNotHealthy11s, controllerManagerPodHealthy, schedulerPodHealthy, etcdPodHealthy, etcdMemberHealthy}}}}, + ), + EtcdMembers: []*etcd.Member{}, + EtcdMembersAgreeOnMemberList: true, + EtcdMembersAgreeOnClusterID: true, + EtcdMembersAndMachinesAreMatching: true, + }, + expectCondition: metav1.Condition{ + Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, + Status: metav1.ConditionTrue, + Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason, + Message: "* 2 of 3 Machines have healthy control plane components, at least 1 required", // two are not healthy, but one just flipped recently and 10s safeguard against flake did not expired yet + }, + }, { name: "One not healthy etcd members, but within quorum", controlPlane: &internal.ControlPlane{ @@ -897,7 +1005,7 @@ func Test_setAvailableCondition(t *testing.T) { Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionFalse, Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, - Message: "Control plane metadata.deletionTimestamp is set", + Message: "* Control plane metadata.deletionTimestamp is set", }, }, { @@ -923,7 +1031,7 @@ func Test_setAvailableCondition(t *testing.T) { Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionFalse, Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, - Message: "Control plane certificates are not available", + Message: "* Control plane certificates are not available", }, }, { @@ -951,7 +1059,7 @@ func Test_setAvailableCondition(t *testing.T) { Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionFalse, Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, - Message: "There is 1 healthy etcd member, at least 2 required for etcd quorum", + Message: "* 1 of 3 etcd members is healthy, at least 2 required for etcd quorum", }, }, { @@ -979,7 +1087,7 @@ func Test_setAvailableCondition(t *testing.T) { Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionFalse, Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, - Message: "There are no Machines with healthy control plane components, at least 1 required", + Message: "* There are no Machines with healthy control plane components, at least 1 required", }, }, { @@ -1016,6 +1124,41 @@ func Test_setAvailableCondition(t *testing.T) { Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason, }, }, + { + name: "External etcd, at least one K8s control plane, some control plane failures to be reported", + controlPlane: &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + Etcd: bootstrapv1.Etcd{External: &bootstrapv1.ExternalEtcd{}}, + }, + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + V1Beta2: &controlplanev1.KubeadmControlPlaneV1Beta2Status{ + Conditions: []metav1.Condition{certificatesReady}, + }, + }, + }, + Machines: collections.FromMachines( + &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodHealthy, controllerManagerPodHealthy, schedulerPodHealthy}}}}, + &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m2"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodNotHealthy, controllerManagerPodHealthy, schedulerPodHealthy}}}}, + &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m3"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{apiServerPodNotHealthy11s, controllerManagerPodHealthy, schedulerPodHealthy}}}}, + ), + EtcdMembers: nil, + EtcdMembersAgreeOnMemberList: false, + EtcdMembersAgreeOnClusterID: false, + EtcdMembersAndMachinesAreMatching: false, + }, + expectCondition: metav1.Condition{ + Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, + Status: metav1.ConditionTrue, + Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason, + Message: "* 2 of 3 Machines have healthy control plane components, at least 1 required", // two are not healthy, but one just flipped recently and 10s safeguard against flake did not expired yet + }, + }, { name: "External etcd, not enough healthy K8s control planes", controlPlane: &internal.ControlPlane{ @@ -1048,7 +1191,7 @@ func Test_setAvailableCondition(t *testing.T) { Type: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition, Status: metav1.ConditionFalse, Reason: controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason, - Message: "There are no Machines with healthy control plane components, at least 1 required", + Message: "* There are no Machines with healthy control plane components, at least 1 required", }, }, }