chore: Add status conditions on nodepool (#1248)

kubernetes-sigs · Jul 5, 2024 · 4a72da3 · 4a72da3
1 parent e356a51
commit 4a72da3
Show file tree

Hide file tree

Showing 46 changed files with 846 additions and 79 deletions.
diff --git a/kwok/cloudprovider/cloudprovider.go b/kwok/cloudprovider/cloudprovider.go
@@ -23,14 +23,13 @@ import (
 	"math/rand"
 	"strings"
 
-	"github.com/awslabs/operatorpkg/object"
+	"github.com/awslabs/operatorpkg/status"
 
 	"github.com/docker/docker/pkg/namesgenerator"
 	"github.com/samber/lo"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -124,8 +123,8 @@ func (c CloudProvider) Name() string {
 	return "kwok"
 }
 
-func (c CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind {
-	return []schema.GroupVersionKind{object.GVK(&v1alpha1.KWOKNodeClass{})}
+func (c CloudProvider) GetSupportedNodeClasses() []status.Object {
+	return []status.Object{&v1alpha1.KWOKNodeClass{}}
 }
 
 func (c CloudProvider) getInstanceType(instanceTypeName string) (*cloudprovider.InstanceType, error) {

diff --git a/pkg/apis/v1/nodeclaim_conversion_test.go b/pkg/apis/v1/nodeclaim_conversion_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package v1_test
 
 import (
+	"github.com/awslabs/operatorpkg/object"
 	"github.com/awslabs/operatorpkg/status"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -26,6 +27,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "sigs.k8s.io/karpenter/pkg/apis/v1"
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/operator/injection"
@@ -88,14 +91,11 @@ var _ = Describe("Convert v1 to v1beta1 NodeClaim API", func() {
 			},
 		}
 		Expect(env.Client.Create(ctx, v1nodepool)).To(Succeed())
-		cloudProvider.NodeClassGroupVersionKind = []schema.GroupVersionKind{
-			{
-				Group:   "fake-cloudprovider-group",
-				Version: "fake-cloudprovider-version",
-				Kind:    "fake-cloudprovider-kind",
-			},
-		}
-		ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses())
+		gvk := lo.Map(cloudProvider.GetSupportedNodeClasses(), func(nc status.Object, _ int) schema.GroupVersionKind {
+			return object.GVK(nc)
+		})
+		cloudProvider.NodeClassGroupVersionKind = gvk
+		ctx = injection.WithNodeClasses(ctx, gvk)
 	})
 
 	It("should convert v1 nodeclaim metadata", func() {
@@ -185,9 +185,9 @@ var _ = Describe("Convert v1 to v1beta1 NodeClaim API", func() {
 		Context("NodeClassRef", func() {
 			It("should convert v1 nodeclaim template nodeClassRef", func() {
 				v1nodeclaim.Spec.NodeClassRef = &NodeClassReference{
-					Kind:  "fake-cloudprovider-kind",
+					Kind:  object.GVK(&v1alpha1.TestNodeClass{}).Kind,
 					Name:  "nodeclass-test",
-					Group: "fake-cloudprovider-group",
+					Group: object.GVK(&v1alpha1.TestNodeClass{}).Group,
 				}
 				Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed())
 				Expect(v1beta1nodeclaim.Spec.NodeClassRef.Kind).To(Equal(v1nodeclaim.Spec.NodeClassRef.Kind))
@@ -312,14 +312,11 @@ var _ = Describe("Convert V1beta1 to V1 NodeClaim API", func() {
 			},
 		}
 		Expect(env.Client.Create(ctx, v1beta1nodepool)).To(Succeed())
-		cloudProvider.NodeClassGroupVersionKind = []schema.GroupVersionKind{
-			{
-				Group:   "fake-cloudprovider-group",
-				Version: "fake-cloudprovider-version",
-				Kind:    "fake-cloudprovider-kind",
-			},
-		}
-		ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses())
+		gvk := lo.Map(cloudProvider.GetSupportedNodeClasses(), func(nc status.Object, _ int) schema.GroupVersionKind {
+			return object.GVK(nc)
+		})
+		cloudProvider.NodeClassGroupVersionKind = gvk
+		ctx = injection.WithNodeClasses(ctx, gvk)
 	})
 
 	It("should convert v1beta1 nodeclaim metadata", func() {

diff --git a/pkg/apis/v1/nodepool_conversion_test.go b/pkg/apis/v1/nodepool_conversion_test.go
@@ -19,6 +19,11 @@ package v1_test
 import (
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
+	"github.com/awslabs/operatorpkg/object"
+	"github.com/awslabs/operatorpkg/status"
+
 	"sigs.k8s.io/karpenter/pkg/test"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -65,14 +70,11 @@ var _ = Describe("Convert V1 to V1beta1 NodePool API", func() {
 				},
 			},
 		}
-		cloudProvider.NodeClassGroupVersionKind = []schema.GroupVersionKind{
-			{
-				Group:   "fake-cloudprovider-group",
-				Version: "fake-cloudprovider-version",
-				Kind:    "fake-cloudprovider-kind",
-			},
-		}
-		ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses())
+		gvk := lo.Map(cloudProvider.GetSupportedNodeClasses(), func(nc status.Object, _ int) schema.GroupVersionKind {
+			return object.GVK(nc)
+		})
+		cloudProvider.NodeClassGroupVersionKind = gvk
+		ctx = injection.WithNodeClasses(ctx, gvk)
 	})
 
 	It("should convert v1 nodepool metadata", func() {
@@ -180,9 +182,9 @@ var _ = Describe("Convert V1 to V1beta1 NodePool API", func() {
 			Context("NodeClassRef", func() {
 				It("should convert v1 nodepool template nodeClassRef", func() {
 					v1nodepool.Spec.Template.Spec.NodeClassRef = &NodeClassReference{
-						Kind:  "fake-cloudprovider-kind",
+						Kind:  object.GVK(&v1alpha1.TestNodeClass{}).Kind,
 						Name:  "nodeclass-test",
-						Group: "fake-cloudprovider-group",
+						Group: object.GVK(&v1alpha1.TestNodeClass{}).Group,
 					}
 					Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed())
 					Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Kind).To(Equal(v1nodepool.Spec.Template.Spec.NodeClassRef.Kind))
@@ -306,14 +308,11 @@ var _ = Describe("Convert V1beta1 to V1 NodePool API", func() {
 				},
 			},
 		}
-		cloudProvider.NodeClassGroupVersionKind = []schema.GroupVersionKind{
-			{
-				Group:   "fake-cloudprovider-group",
-				Version: "fake-cloudprovider-version",
-				Kind:    "fake-cloudprovider-kind",
-			},
-		}
-		ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses())
+		gvk := lo.Map(cloudProvider.GetSupportedNodeClasses(), func(nc status.Object, _ int) schema.GroupVersionKind {
+			return object.GVK(nc)
+		})
+		cloudProvider.NodeClassGroupVersionKind = gvk
+		ctx = injection.WithNodeClasses(ctx, gvk)
 	})
 
 	It("should convert v1beta1 nodepool metadata", func() {

diff --git a/pkg/apis/v1/nodepool_status.go b/pkg/apis/v1/nodepool_status.go
@@ -25,6 +25,8 @@ const (
 	// ConditionTypeValidationSucceeded = "ValidationSucceeded" condition indicates that the
 	// runtime-based configuration is valid for this NodePool
 	ConditionTypeValidationSucceeded = "ValidationSucceeded"
+	// ConditionTypeNodeClassReady = "NodeClassReady" condition indicates that underlying nodeClass was resolved and is reporting as Ready
+	ConditionTypeNodeClassReady = "NodeClassReady"
 )
 
 // NodePoolStatus defines the observed state of NodePool
@@ -38,7 +40,10 @@ type NodePoolStatus struct {
 }
 
 func (in *NodePool) StatusConditions() status.ConditionSet {
-	return status.NewReadyConditions(ConditionTypeValidationSucceeded).For(in)
+	return status.NewReadyConditions(
+		ConditionTypeValidationSucceeded,
+		ConditionTypeNodeClassReady,
+	).For(in)
 }
 
 func (in *NodePool) GetConditions() []status.Condition {

diff --git a/pkg/apis/v1/suite_test.go b/pkg/apis/v1/suite_test.go
@@ -21,6 +21,8 @@ import (
 	"math/rand"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
@@ -45,7 +47,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 	cloudProvider = fake.NewCloudProvider()
 })
 

diff --git a/pkg/apis/v1beta1/nodepool_status.go b/pkg/apis/v1beta1/nodepool_status.go
@@ -25,6 +25,8 @@ const (
 	// ConditionTypeValidationSucceeded = "ValidationSucceeded" condition indicates that the
 	// runtime-based configuration is valid for this NodePool
 	ConditionTypeValidationSucceeded = "ValidationSucceeded"
+	// ConditionTypeNodeClassReady = "NodeClassReady" condition indicates that underlying nodeClass was resolved and is reporting as Ready
+	ConditionTypeNodeClassReady = "NodeClassReady"
 )
 
 // NodePoolStatus defines the observed state of NodePool
@@ -38,7 +40,10 @@ type NodePoolStatus struct {
 }
 
 func (in *NodePool) StatusConditions() status.ConditionSet {
-	return status.NewReadyConditions(ConditionTypeValidationSucceeded).For(in)
+	return status.NewReadyConditions(
+		ConditionTypeValidationSucceeded,
+		ConditionTypeNodeClassReady,
+	).For(in)
 }
 
 func (in *NodePool) GetConditions() []status.Condition {

diff --git a/pkg/apis/v1beta1/suite_test.go b/pkg/apis/v1beta1/suite_test.go
@@ -21,6 +21,8 @@ import (
 	"math/rand"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
@@ -43,7 +45,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 })
 
 var _ = AfterEach(func() {

diff --git a/pkg/cloudprovider/fake/cloudprovider.go b/pkg/cloudprovider/fake/cloudprovider.go
@@ -23,6 +23,10 @@ import (
 	"sort"
 	"sync"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
+	"github.com/awslabs/operatorpkg/status"
+
 	"github.com/samber/lo"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -262,6 +266,6 @@ func (c *CloudProvider) Name() string {
 	return "fake"
 }
 
-func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind {
-	return c.NodeClassGroupVersionKind
+func (c *CloudProvider) GetSupportedNodeClasses() []status.Object {
+	return []status.Object{&v1alpha1.TestNodeClass{}}
 }
diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go
@@ -24,9 +24,10 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/awslabs/operatorpkg/status"
+
 	"github.com/samber/lo"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
@@ -62,8 +63,9 @@ type CloudProvider interface {
 	IsDrifted(context.Context, *v1beta1.NodeClaim) (DriftReason, error)
 	// Name returns the CloudProvider implementation name.
 	Name() string
-	// GetSupportedNodeClass returns the group, version, and kind of the CloudProvider NodeClass
-	GetSupportedNodeClasses() []schema.GroupVersionKind
+	// GetSupportedNodeClasses returns CloudProvider NodeClass that implements status.Object
+	// NOTE: It returns a list where the first element should be the default NodeClass
+	GetSupportedNodeClasses() []status.Object
 }
 
 // InstanceType describes the properties of a potential node (either concrete attributes of an instance of this type

diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go
@@ -21,6 +21,8 @@ import (
 	"k8s.io/utils/clock"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	nodepoolreadiness "sigs.k8s.io/karpenter/pkg/controllers/nodepool/readiness"
+
 	"sigs.k8s.io/karpenter/pkg/cloudprovider"
 	"sigs.k8s.io/karpenter/pkg/controllers/disruption"
 	"sigs.k8s.io/karpenter/pkg/controllers/disruption/orchestration"
@@ -71,6 +73,7 @@ func NewControllers(
 		metricspod.NewController(kubeClient),
 		metricsnodepool.NewController(kubeClient),
 		metricsnode.NewController(cluster),
+		nodepoolreadiness.NewController(kubeClient, cloudProvider),
 		nodepoolcounter.NewController(kubeClient, cluster),
 		nodepoolvalidation.NewController(kubeClient),
 		nodeclaimconsistency.NewController(clock, kubeClient, recorder),

diff --git a/pkg/controllers/disruption/orchestration/suite_test.go b/pkg/controllers/disruption/orchestration/suite_test.go
@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -70,7 +72,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 	ctx = options.ToContext(ctx, test.Options())
 	fakeClock = clock.NewFakeClock(time.Now())
 	cloudProvider = fake.NewCloudProvider()

diff --git a/pkg/controllers/disruption/suite_test.go b/pkg/controllers/disruption/suite_test.go
@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
@@ -80,7 +82,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(coreapis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(coreapis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 	ctx = options.ToContext(ctx, test.Options())
 	cloudProvider = fake.NewCloudProvider()
 	fakeClock = clock.NewFakeClock(time.Now())

diff --git a/pkg/controllers/leasegarbagecollection/suite_test.go b/pkg/controllers/leasegarbagecollection/suite_test.go
@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/samber/lo"
 	coordinationsv1 "k8s.io/api/coordination/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -49,7 +51,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 	ctx = options.ToContext(ctx, test.Options())
 
 	garbageCollectionController = leasegarbagecollection.NewController(env.Client)

diff --git a/pkg/controllers/metrics/node/suite_test.go b/pkg/controllers/metrics/node/suite_test.go
@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	clock "k8s.io/utils/clock/testing"
 
 	v1 "k8s.io/api/core/v1"
@@ -59,7 +61,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 
 	ctx = options.ToContext(ctx, test.Options())
 	cloudProvider = fake.NewCloudProvider()

diff --git a/pkg/controllers/metrics/nodepool/suite_test.go b/pkg/controllers/metrics/nodepool/suite_test.go
@@ -21,6 +21,8 @@ import (
 	"strings"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/api/core/v1"
@@ -47,7 +49,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...))
+	env = test.NewEnvironment(test.WithCRDs(apis.CRDs...), test.WithCRDs(v1alpha1.CRDs...))
 	nodePoolController = nodepool.NewController(env.Client)
 })