diff --git a/api/v1alpha1/rollingupgrade_types.go b/api/v1alpha1/rollingupgrade_types.go index 7b5b27b5..8526be50 100644 --- a/api/v1alpha1/rollingupgrade_types.go +++ b/api/v1alpha1/rollingupgrade_types.go @@ -52,6 +52,12 @@ type RollingUpgradeSpec struct { IgnoreDrainFailures bool `json:"ignoreDrainFailures,omitempty"` // ForceRefresh enables draining and terminating the node even if the launch config/template hasn't changed. ForceRefresh bool `json:"forceRefresh,omitempty"` + // ReadinessGates allow to specify label selectors that node must match to be considered ready. + ReadinessGates []NodeReadinessGate `json:"readinessGates,omitempty"` +} + +type NodeReadinessGate struct { + MatchLabels map[string]string `json:"matchLabels,omitempty" protobuf:"bytes,1,rep,name=matchLabels"` } // RollingUpgradeStatus defines the observed state of RollingUpgrade diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 72ab3911..3f4d426b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,6 +23,28 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeReadinessGate) DeepCopyInto(out *NodeReadinessGate) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeReadinessGate. +func (in *NodeReadinessGate) DeepCopy() *NodeReadinessGate { + if in == nil { + return nil + } + out := new(NodeReadinessGate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PostDrainSpec) DeepCopyInto(out *PostDrainSpec) { *out = *in @@ -73,7 +95,7 @@ func (in *RollingUpgrade) DeepCopyInto(out *RollingUpgrade) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -149,6 +171,13 @@ func (in *RollingUpgradeSpec) DeepCopyInto(out *RollingUpgradeSpec) { out.PostDrain = in.PostDrain out.PostTerminate = in.PostTerminate out.Strategy = in.Strategy + if in.ReadinessGates != nil { + in, out := &in.ReadinessGates, &out.ReadinessGates + *out = make([]NodeReadinessGate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpgradeSpec. diff --git a/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml b/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml index bffc937e..48339071 100644 --- a/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml +++ b/config/crd/bases/upgrademgr.keikoproj.io_rollingupgrades.yaml @@ -92,6 +92,17 @@ spec: script: type: string type: object + readinessGates: + description: ReadinessGates allow to specify label selectors that node + must match to be considered ready. + items: + properties: + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array strategy: description: UpdateStrategy holds the information needed to perform update based on different update strategies diff --git a/controllers/helpers.go b/controllers/helpers.go index 3b8a09b6..e3ead121 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -50,6 +50,21 @@ func isNodeReady(node corev1.Node) bool { return false } +func IsNodePassesReadinessGates(node corev1.Node, requiredReadinessGates []upgrademgrv1alpha1.NodeReadinessGate) bool { + + if len(requiredReadinessGates) == 0 { + return true + } + for _, gate := range requiredReadinessGates { + for key, value := range gate.MatchLabels { + if node.Labels[key] != value { + return false + } + } + } + return true +} + func getInServiceCount(instances []*autoscaling.Instance) int64 { var count int64 for _, instance := range instances { diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go index a8f828b9..bda85f8d 100644 --- a/controllers/helpers_test.go +++ b/controllers/helpers_test.go @@ -3,6 +3,7 @@ package controllers import ( "encoding/json" "fmt" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" @@ -45,6 +46,105 @@ func TestIsNodeReady(t *testing.T) { } } +func TestIsNodePassesReadinessGates(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + type test struct { + gate []map[string]string + labels map[string]string + want bool + } + tests := []test{ + { + gate: []map[string]string{ + { + "healthy": "true", + }, + }, + labels: map[string]string{ + "healthy": "true", + }, + want: true, + }, + + { + gate: []map[string]string{}, + labels: map[string]string{ + "healthy": "true", + }, + want: true, + }, + + { + gate: []map[string]string{ + {"healthy": "true"}, + }, + labels: map[string]string{ + "healthy": "false", + }, + want: false, + }, + + { + gate: []map[string]string{ + {"healthy": "true"}, + }, + labels: map[string]string{ + + }, + want: false, + }, + + { + gate: []map[string]string{ + {"healthy": "true"}, + {"second-check": "true"}, + }, + labels: map[string]string{ + "healthy": "true", + }, + want: false, + }, + { + gate: []map[string]string{ + {"healthy": "true"}, + {"second-check": "true"}, + }, + labels: map[string]string{ + "healthy": "true", + "second-check": "true", + }, + want: true, + }, + { + gate: []map[string]string{ + {"healthy": "true"}, + {"second-check": "true"}, + }, + labels: map[string]string{ + "healthy": "true", + "second-check": "false", + }, + want: false, + }, } + + for _, tt := range tests { + readinessGates := make([]upgrademgrv1alpha1.NodeReadinessGate, len(tt.gate)) + for i, g := range tt.gate { + readinessGates[i] = upgrademgrv1alpha1.NodeReadinessGate{ + MatchLabels: g, + } + } + node := corev1.Node{ + ObjectMeta: v1.ObjectMeta{ + Labels: tt.labels, + }, + } + g.Expect(IsNodePassesReadinessGates(node, readinessGates)).To(gomega.Equal(tt.want)) + } + +} + func TestGetInServiceCount(t *testing.T) { g := gomega.NewGomegaWithT(t) tt := map[*autoscaling.Instance]int64{ diff --git a/controllers/rollingupgrade_controller.go b/controllers/rollingupgrade_controller.go index 689a31e1..f994823e 100644 --- a/controllers/rollingupgrade_controller.go +++ b/controllers/rollingupgrade_controller.go @@ -308,7 +308,7 @@ func (r *RollingUpgradeReconciler) WaitForDesiredNodes(ruObj *upgrademgrv1alpha1 for _, node := range r.NodeList.Items { tokens := strings.Split(node.Spec.ProviderID, "/") instanceID := tokens[len(tokens)-1] - if contains(inServiceInstances, instanceID) && isNodeReady(node) { + if contains(inServiceInstances, instanceID) && isNodeReady(node) && IsNodePassesReadinessGates(node, ruObj.Spec.ReadinessGates) { foundCount++ } }