From 4225fe015d0d1e0f60a8ba457643a37981304d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Thu, 14 Nov 2024 19:05:28 +0800 Subject: [PATCH] feat(kcc): canary update --- ...t.kubewharf.io_adminqosconfigurations.yaml | 66 +++++++++++++++++++ ...alyst.kubewharf.io_authconfigurations.yaml | 66 +++++++++++++++++++ ...sparentmemoryoffloadingconfigurations.yaml | 66 +++++++++++++++++++ pkg/apis/config/v1alpha1/adminqos.go | 5 ++ pkg/apis/config/v1alpha1/auth.go | 5 ++ pkg/apis/config/v1alpha1/generic.go | 63 ++++++++++++++++++ pkg/apis/config/v1alpha1/tmo.go | 5 ++ .../config/v1alpha1/zz_generated.deepcopy.go | 44 +++++++++++++ 8 files changed, 320 insertions(+) diff --git a/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml b/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml index 53f21339..31fa0a3f 100644 --- a/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml +++ b/config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml @@ -21,6 +21,9 @@ spec: - jsonPath: .metadata.creationTimestamp name: AGE type: date + - jsonPath: .spec.paused + name: PAUSED + type: boolean - jsonPath: .spec.nodeLabelSelector name: SELECTOR type: string @@ -33,6 +36,18 @@ spec: - jsonPath: .spec.ephemeralSelector.lastDuration name: DURATION type: string + - jsonPath: .status.targetNodes + name: TARGET + type: integer + - jsonPath: .status.canaryNodes + name: CANARY + type: integer + - jsonPath: .status.updatedTargetNodes + name: UPDATED-TARGET + type: integer + - jsonPath: .status.currentHash + name: HASH + type: string - jsonPath: .status.conditions[?(@.type=="Valid")].status name: VALID type: string @@ -680,6 +695,9 @@ spec: KatalystCustomConfig.spec.nodeLabelSelectorAllowedKeyList, otherwise it will not be synced. type: string + paused: + description: Indicates that the config is paused. + type: boolean priority: description: Priority is used by one node matched by NodeLabelSelector of more than one configuration, and the higher priority will be @@ -695,11 +713,42 @@ spec: currently applied Spec version. The default value is 3. format: int64 type: integer + updateStrategy: + description: An update strategy to replace existing CustomNodeConfig + configurations with new ones. + properties: + rollingUpdate: + description: 'Rolling update config params. Present only if type + = "RollingUpdate". --- TODO: Update this to follow our convention + for oneOf, whatever we decide it to be. Same as Deployment `strategy.rollingUpdate`. + See https://github.com/kubernetes/kubernetes/issues/35345' + properties: + canary: + anyOf: + - type: integer + - type: string + description: 'The number or percentage of target CustomNodeConfigs + to update to the current configuration. For example: `100`` + and `20%` are valid values.' + pattern: ^(100|[1-9][0-9]?|0)%$ + x-kubernetes-int-or-string: true + type: object + type: + description: Type of config update. Only `RollingUpdate` is supported. + enum: + - RollingUpdate + type: string + type: object required: - config type: object status: properties: + canaryNodes: + description: The number of nodes that this config is targeting and + should be updated given the current strategy. + format: int32 + type: integer collisionCount: description: Count of hash collisions for this cr. The kcc controller uses this field as a collision avoidance mechanism when it needs @@ -734,10 +783,27 @@ spec: - type type: object type: array + currentHash: + description: The hash of the current config observed by the kcc controller. + type: string observedGeneration: description: The most recent generation observed by the kcc controller. format: int64 type: integer + targetNodes: + description: The number of nodes that this config is targeting. + format: int32 + type: integer + updatedNodes: + description: The number of nodes (including non-target nodes) that + have been updated by this config. + format: int32 + type: integer + updatedTargetNodes: + description: The number of target nodes that have been updated by + this config. + format: int32 + type: integer type: object type: object served: true diff --git a/config/crd/bases/config.katalyst.kubewharf.io_authconfigurations.yaml b/config/crd/bases/config.katalyst.kubewharf.io_authconfigurations.yaml index cca9d34d..fcdc0f8b 100644 --- a/config/crd/bases/config.katalyst.kubewharf.io_authconfigurations.yaml +++ b/config/crd/bases/config.katalyst.kubewharf.io_authconfigurations.yaml @@ -21,6 +21,9 @@ spec: - jsonPath: .metadata.creationTimestamp name: AGE type: date + - jsonPath: .spec.paused + name: PAUSED + type: boolean - jsonPath: .spec.nodeLabelSelector name: SELECTOR type: string @@ -33,6 +36,18 @@ spec: - jsonPath: .spec.ephemeralSelector.lastDuration name: DURATION type: string + - jsonPath: .status.targetNodes + name: TARGET + type: integer + - jsonPath: .status.canaryNodes + name: CANARY + type: integer + - jsonPath: .status.updatedTargetNodes + name: UPDATED-TARGET + type: integer + - jsonPath: .status.currentHash + name: HASH + type: string - jsonPath: .status.conditions[?(@.type=="Valid")].status name: VALID type: string @@ -121,6 +136,9 @@ spec: KatalystCustomConfig.spec.nodeLabelSelectorAllowedKeyList, otherwise it will not be synced. type: string + paused: + description: Indicates that the config is paused. + type: boolean priority: description: Priority is used by one node matched by NodeLabelSelector of more than one configuration, and the higher priority will be @@ -136,9 +154,40 @@ spec: currently applied Spec version. The default value is 3. format: int64 type: integer + updateStrategy: + description: An update strategy to replace existing CustomNodeConfig + configurations with new ones. + properties: + rollingUpdate: + description: 'Rolling update config params. Present only if type + = "RollingUpdate". --- TODO: Update this to follow our convention + for oneOf, whatever we decide it to be. Same as Deployment `strategy.rollingUpdate`. + See https://github.com/kubernetes/kubernetes/issues/35345' + properties: + canary: + anyOf: + - type: integer + - type: string + description: 'The number or percentage of target CustomNodeConfigs + to update to the current configuration. For example: `100`` + and `20%` are valid values.' + pattern: ^(100|[1-9][0-9]?|0)%$ + x-kubernetes-int-or-string: true + type: object + type: + description: Type of config update. Only `RollingUpdate` is supported. + enum: + - RollingUpdate + type: string + type: object type: object status: properties: + canaryNodes: + description: The number of nodes that this config is targeting and + should be updated given the current strategy. + format: int32 + type: integer collisionCount: description: Count of hash collisions for this cr. The kcc controller uses this field as a collision avoidance mechanism when it needs @@ -173,10 +222,27 @@ spec: - type type: object type: array + currentHash: + description: The hash of the current config observed by the kcc controller. + type: string observedGeneration: description: The most recent generation observed by the kcc controller. format: int64 type: integer + targetNodes: + description: The number of nodes that this config is targeting. + format: int32 + type: integer + updatedNodes: + description: The number of nodes (including non-target nodes) that + have been updated by this config. + format: int32 + type: integer + updatedTargetNodes: + description: The number of target nodes that have been updated by + this config. + format: int32 + type: integer type: object type: object served: true diff --git a/config/crd/bases/config.katalyst.kubewharf.io_transparentmemoryoffloadingconfigurations.yaml b/config/crd/bases/config.katalyst.kubewharf.io_transparentmemoryoffloadingconfigurations.yaml index 1641e44d..f7a16516 100644 --- a/config/crd/bases/config.katalyst.kubewharf.io_transparentmemoryoffloadingconfigurations.yaml +++ b/config/crd/bases/config.katalyst.kubewharf.io_transparentmemoryoffloadingconfigurations.yaml @@ -21,6 +21,9 @@ spec: - jsonPath: .metadata.creationTimestamp name: AGE type: date + - jsonPath: .spec.paused + name: PAUSED + type: boolean - jsonPath: .spec.nodeLabelSelector name: SELECTOR type: string @@ -33,6 +36,18 @@ spec: - jsonPath: .spec.ephemeralSelector.lastDuration name: DURATION type: string + - jsonPath: .status.targetNodes + name: TARGET + type: integer + - jsonPath: .status.canaryNodes + name: CANARY + type: integer + - jsonPath: .status.updatedTargetNodes + name: UPDATED-TARGET + type: integer + - jsonPath: .status.currentHash + name: HASH + type: string - jsonPath: .status.conditions[?(@.type=="Valid")].status name: VALID type: string @@ -295,6 +310,9 @@ spec: KatalystCustomConfig.spec.nodeLabelSelectorAllowedKeyList, otherwise it will not be synced. type: string + paused: + description: Indicates that the config is paused. + type: boolean priority: description: Priority is used by one node matched by NodeLabelSelector of more than one configuration, and the higher priority will be @@ -310,11 +328,42 @@ spec: currently applied Spec version. The default value is 3. format: int64 type: integer + updateStrategy: + description: An update strategy to replace existing CustomNodeConfig + configurations with new ones. + properties: + rollingUpdate: + description: 'Rolling update config params. Present only if type + = "RollingUpdate". --- TODO: Update this to follow our convention + for oneOf, whatever we decide it to be. Same as Deployment `strategy.rollingUpdate`. + See https://github.com/kubernetes/kubernetes/issues/35345' + properties: + canary: + anyOf: + - type: integer + - type: string + description: 'The number or percentage of target CustomNodeConfigs + to update to the current configuration. For example: `100`` + and `20%` are valid values.' + pattern: ^(100|[1-9][0-9]?|0)%$ + x-kubernetes-int-or-string: true + type: object + type: + description: Type of config update. Only `RollingUpdate` is supported. + enum: + - RollingUpdate + type: string + type: object required: - config type: object status: properties: + canaryNodes: + description: The number of nodes that this config is targeting and + should be updated given the current strategy. + format: int32 + type: integer collisionCount: description: Count of hash collisions for this cr. The kcc controller uses this field as a collision avoidance mechanism when it needs @@ -349,10 +398,27 @@ spec: - type type: object type: array + currentHash: + description: The hash of the current config observed by the kcc controller. + type: string observedGeneration: description: The most recent generation observed by the kcc controller. format: int64 type: integer + targetNodes: + description: The number of nodes that this config is targeting. + format: int32 + type: integer + updatedNodes: + description: The number of nodes (including non-target nodes) that + have been updated by this config. + format: int32 + type: integer + updatedTargetNodes: + description: The number of target nodes that have been updated by + this config. + format: int32 + type: integer type: object type: object served: true diff --git a/pkg/apis/config/v1alpha1/adminqos.go b/pkg/apis/config/v1alpha1/adminqos.go index d412d539..64553fa8 100644 --- a/pkg/apis/config/v1alpha1/adminqos.go +++ b/pkg/apis/config/v1alpha1/adminqos.go @@ -29,10 +29,15 @@ import ( // +kubebuilder:resource:path=adminqosconfigurations,shortName=aqc // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=.metadata.creationTimestamp +// +kubebuilder:printcolumn:name="PAUSED",type=boolean,JSONPath=".spec.paused" // +kubebuilder:printcolumn:name="SELECTOR",type=string,JSONPath=".spec.nodeLabelSelector" // +kubebuilder:printcolumn:name="PRIORITY",type=string,JSONPath=".spec.priority" // +kubebuilder:printcolumn:name="NODES",type=string,JSONPath=".spec.ephemeralSelector.nodeNames" // +kubebuilder:printcolumn:name="DURATION",type=string,JSONPath=".spec.ephemeralSelector.lastDuration" +// +kubebuilder:printcolumn:name="TARGET",type=integer,JSONPath=".status.targetNodes" +// +kubebuilder:printcolumn:name="CANARY",type=integer,JSONPath=".status.canaryNodes" +// +kubebuilder:printcolumn:name="UPDATED-TARGET",type=integer,JSONPath=".status.updatedTargetNodes" +// +kubebuilder:printcolumn:name="HASH",type=string,JSONPath=".status.currentHash" // +kubebuilder:printcolumn:name="VALID",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].status" // +kubebuilder:printcolumn:name="REASON",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].reason" // +kubebuilder:printcolumn:name="MESSAGE",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].message" diff --git a/pkg/apis/config/v1alpha1/auth.go b/pkg/apis/config/v1alpha1/auth.go index 80301b87..04a28503 100644 --- a/pkg/apis/config/v1alpha1/auth.go +++ b/pkg/apis/config/v1alpha1/auth.go @@ -23,10 +23,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // +kubebuilder:resource:path=authconfigurations,shortName=ac // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=.metadata.creationTimestamp +// +kubebuilder:printcolumn:name="PAUSED",type=boolean,JSONPath=".spec.paused" // +kubebuilder:printcolumn:name="SELECTOR",type=string,JSONPath=".spec.nodeLabelSelector" // +kubebuilder:printcolumn:name="PRIORITY",type=string,JSONPath=".spec.priority" // +kubebuilder:printcolumn:name="NODES",type=string,JSONPath=".spec.ephemeralSelector.nodeNames" // +kubebuilder:printcolumn:name="DURATION",type=string,JSONPath=".spec.ephemeralSelector.lastDuration" +// +kubebuilder:printcolumn:name="TARGET",type=integer,JSONPath=".status.targetNodes" +// +kubebuilder:printcolumn:name="CANARY",type=integer,JSONPath=".status.canaryNodes" +// +kubebuilder:printcolumn:name="UPDATED-TARGET",type=integer,JSONPath=".status.updatedTargetNodes" +// +kubebuilder:printcolumn:name="HASH",type=string,JSONPath=".status.currentHash" // +kubebuilder:printcolumn:name="VALID",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].status" // +kubebuilder:printcolumn:name="REASON",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].reason" // +kubebuilder:printcolumn:name="MESSAGE",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].message" diff --git a/pkg/apis/config/v1alpha1/generic.go b/pkg/apis/config/v1alpha1/generic.go index be338c89..e3533fba 100644 --- a/pkg/apis/config/v1alpha1/generic.go +++ b/pkg/apis/config/v1alpha1/generic.go @@ -19,6 +19,7 @@ package v1alpha1 import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) type GenericConfigSpec struct { @@ -45,6 +46,47 @@ type GenericConfigSpec struct { // EphemeralSelector is a selector for temporary use only // +optional EphemeralSelector EphemeralSelector `json:"ephemeralSelector,omitempty"` + + // Indicates that the config is paused. + // +optional + Paused bool `json:"paused,omitempty"` + + // An update strategy to replace existing CustomNodeConfig configurations with new ones. + // +optional + UpdateStrategy ConfigUpdateStrategy `json:"updateStrategy,omitempty"` +} + +// ConfigUpdateStrategy is a struct used to control the update strategy for a KatalystCustomConfig target. +type ConfigUpdateStrategy struct { + // Type of config update. Only `RollingUpdate` is supported. + // +optional + Type ConfigUpdateStrategyType `json:"type,omitempty"` + + // Rolling update config params. Present only if type = "RollingUpdate". + //--- + // TODO: Update this to follow our convention for oneOf, whatever we decide it + // to be. Same as Deployment `strategy.rollingUpdate`. + // See https://github.com/kubernetes/kubernetes/issues/35345 + // +optional + RollingUpdate *RollingUpdateConfig `json:"rollingUpdate,omitempty"` +} + +// +kubebuilder:validation:Enum=RollingUpdate +type ConfigUpdateStrategyType string + +const ( + // Replace the old configs by new ones using rolling update i.e replace them on each node one after the other. + RollingUpdateConfigStrategyType ConfigUpdateStrategyType = "RollingUpdate" +) + +// Spec to control the desired behavior of config rolling update. +type RollingUpdateConfig struct { + // The number or percentage of target CustomNodeConfigs to update to the current configuration. + // For example: `100`` and `20%` are valid values. + // +kubebuilder:validation:XIntOrString + // +kubebuilder:validation:Pattern=`^(100|[1-9][0-9]?|0)%$` + // +optional + Canary *intstr.IntOrString `json:"canary,omitempty"` } type EphemeralSelector struct { @@ -58,6 +100,26 @@ type EphemeralSelector struct { } type GenericConfigStatus struct { + // The number of nodes that this config is targeting. + // +optional + TargetNodes int32 `json:"targetNodes"` + + // The number of nodes that this config is targeting and should be updated given the current strategy. + // +optional + CanaryNodes int32 `json:"canaryNodes"` + + // The number of target nodes that have been updated by this config. + // +optional + UpdatedTargetNodes int32 `json:"updatedTargetNodes"` + + // The number of nodes (including non-target nodes) that have been updated by this config. + // +optional + UpdatedNodes int32 `json:"updatedNodes"` + + // The hash of the current config observed by the kcc controller. + // +optional + CurrentHash string `json:"currentHash,omitempty"` + // Count of hash collisions for this cr. The kcc controller // uses this field as a collision avoidance mechanism when it needs to // create the name for the newest ControllerRevision. @@ -69,6 +131,7 @@ type GenericConfigStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` // Represents the latest available observations of a config's current state. + // +optional Conditions []GenericConfigCondition `json:"conditions,omitempty"` } diff --git a/pkg/apis/config/v1alpha1/tmo.go b/pkg/apis/config/v1alpha1/tmo.go index 850025d5..562b17c8 100644 --- a/pkg/apis/config/v1alpha1/tmo.go +++ b/pkg/apis/config/v1alpha1/tmo.go @@ -26,10 +26,15 @@ import ( // +kubebuilder:resource:path=transparentmemoryoffloadingconfigurations,shortName=tmo // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=.metadata.creationTimestamp +// +kubebuilder:printcolumn:name="PAUSED",type=boolean,JSONPath=".spec.paused" // +kubebuilder:printcolumn:name="SELECTOR",type=string,JSONPath=".spec.nodeLabelSelector" // +kubebuilder:printcolumn:name="PRIORITY",type=string,JSONPath=".spec.priority" // +kubebuilder:printcolumn:name="NODES",type=string,JSONPath=".spec.ephemeralSelector.nodeNames" // +kubebuilder:printcolumn:name="DURATION",type=string,JSONPath=".spec.ephemeralSelector.lastDuration" +// +kubebuilder:printcolumn:name="TARGET",type=integer,JSONPath=".status.targetNodes" +// +kubebuilder:printcolumn:name="CANARY",type=integer,JSONPath=".status.canaryNodes" +// +kubebuilder:printcolumn:name="UPDATED-TARGET",type=integer,JSONPath=".status.updatedTargetNodes" +// +kubebuilder:printcolumn:name="HASH",type=string,JSONPath=".status.currentHash" // +kubebuilder:printcolumn:name="VALID",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].status" // +kubebuilder:printcolumn:name="REASON",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].reason" // +kubebuilder:printcolumn:name="MESSAGE",type=string,JSONPath=".status.conditions[?(@.type==\"Valid\")].message" diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 88fbd487..79eac173 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,7 @@ import ( resource "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + intstr "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -671,6 +672,27 @@ func (in *CgroupConfig) DeepCopy() *CgroupConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigUpdateStrategy) DeepCopyInto(out *ConfigUpdateStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateConfig) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigUpdateStrategy. +func (in *ConfigUpdateStrategy) DeepCopy() *ConfigUpdateStrategy { + if in == nil { + return nil + } + out := new(ConfigUpdateStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlKnobConstraints) DeepCopyInto(out *ControlKnobConstraints) { *out = *in @@ -889,6 +911,7 @@ func (in *GenericConfigCondition) DeepCopy() *GenericConfigCondition { func (in *GenericConfigSpec) DeepCopyInto(out *GenericConfigSpec) { *out = *in in.EphemeralSelector.DeepCopyInto(&out.EphemeralSelector) + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) return } @@ -1669,6 +1692,27 @@ func (in *RestrictConstraints) DeepCopy() *RestrictConstraints { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdateConfig) DeepCopyInto(out *RollingUpdateConfig) { + *out = *in + if in.Canary != nil { + in, out := &in.Canary, &out.Canary + *out = new(intstr.IntOrString) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateConfig. +func (in *RollingUpdateConfig) DeepCopy() *RollingUpdateConfig { + if in == nil { + return nil + } + out := new(RollingUpdateConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RootfsPressureEvictionConfig) DeepCopyInto(out *RootfsPressureEvictionConfig) { *out = *in