diff --git a/go.mod b/go.mod index a729bfbe..951c87ae 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( k8s.io/kubelet v0.24.6 k8s.io/kubernetes v1.24.6 k8s.io/metrics v0.24.6 + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed sigs.k8s.io/yaml v1.2.0 ) @@ -59,7 +60,6 @@ require ( k8s.io/component-base v0.24.6 // indirect k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/pkg/apis/scheduling/config/register.go b/pkg/apis/scheduling/config/register.go index 783f6d35..87c77b7d 100644 --- a/pkg/apis/scheduling/config/register.go +++ b/pkg/apis/scheduling/config/register.go @@ -35,6 +35,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { &QoSAwareNodeResourcesFitArgs{}, &QoSAwareNodeResourcesBalancedAllocationArgs{}, &NodeResourceTopologyArgs{}, + &LoadAwareArgs{}, ) return nil } diff --git a/pkg/apis/scheduling/config/types.go b/pkg/apis/scheduling/config/types.go index c8adbcb3..b1b7a1fc 100644 --- a/pkg/apis/scheduling/config/types.go +++ b/pkg/apis/scheduling/config/types.go @@ -15,6 +15,7 @@ package config import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -85,3 +86,37 @@ type ScoringStrategy struct { // Arguments specific to RequestedToCapacityRatio strategy. ReclaimedRequestedToCapacityRatio *kubeschedulerconfig.RequestedToCapacityRatioParam `json:"reclaimedRequestedToCapacityRatio,omitempty"` } + +// IndicatorType indicator participate in calculate score +type IndicatorType string + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// LoadAwareArgs holds arguments used to configure the LoadAwareScheduling plugin. +type LoadAwareArgs struct { + metav1.TypeMeta + + EnablePortrait *bool `json:"enablePortrait,omitempty"` + PodAnnotationLoadAwareEnable *string `json:"podAnnotationLoadAwareEnable,omitempty"` + // FilterExpiredNodeMetrics indicates whether to filter nodes where fails to update NPD. + FilterExpiredNodeMetrics *bool `json:"filterExpiredNodeMetrics,omitempty"` + // NodeMetricsExpiredSeconds indicates the NodeMetrics in NPD expiration in seconds. + // When NodeMetrics expired, the node is considered abnormal. + // default 5 minute + NodeMetricsExpiredSeconds *int64 `json:"NodeMetricsExpiredSeconds,omitempty"` + // ResourceToWeightMap contains resource name and weight. + ResourceToWeightMap map[corev1.ResourceName]int64 `json:"resourceToWeightMap,omitempty"` + // ResourceToThresholdMap contains resource name and threshold. Node can not be scheduled + // if usage of it is more than threshold. + ResourceToThresholdMap map[corev1.ResourceName]int64 `json:"resourceToThresholdMap,omitempty"` + // ResourceToScalingFactorMap contains resource name and scaling factor, which are used to estimate pod usage + // if usage of pod is not exists in node monitor. + ResourceToScalingFactorMap map[corev1.ResourceName]int64 `json:"resourceToScalingFactorMap,omitempty"` + // ResourceToTargetMap contains resource name and node usage target which are used in loadAware score + ResourceToTargetMap map[corev1.ResourceName]int64 `json:"resourceToTargetMap,omitempty"` + // CalculateIndicatorWeight indicates the participates in calculate indicator weight + // The default avg_15min 30, max_1hour 30, max_1day 40 + CalculateIndicatorWeight map[IndicatorType]int64 `json:"calculateIndicatorWeight,omitempty"` + + KubeConfigPath string `json:"kubeConfigPath,omitempty"` +} diff --git a/pkg/apis/scheduling/config/v1beta3/defaults.go b/pkg/apis/scheduling/config/v1beta3/defaults.go index b43d0429..ba16223f 100644 --- a/pkg/apis/scheduling/config/v1beta3/defaults.go +++ b/pkg/apis/scheduling/config/v1beta3/defaults.go @@ -19,6 +19,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/kube-scheduler/config/v1beta3" + "k8s.io/utils/pointer" "github.com/kubewharf/katalyst-api/pkg/consts" ) @@ -35,6 +36,33 @@ var defaultReclaimedResourceSpec = []v1beta3.ResourceSpec{ var defaultAlignedResourceSpec = []string{v1.ResourceCPU.String(), v1.ResourceMemory.String()} +var ( + defaultNodeMonitorExpiredSeconds int64 = 180 + defaultResourceToWeightMap = map[v1.ResourceName]int64{ + v1.ResourceCPU: 1, + v1.ResourceMemory: 1, + } + + defaultResourceToThresholdMap = map[v1.ResourceName]int64{ + v1.ResourceCPU: 70, // 70% + v1.ResourceMemory: 95, // 95% + } + + defaultResourceToScalingFactorMap = map[v1.ResourceName]int64{ + v1.ResourceCPU: 85, // 85% + v1.ResourceMemory: 70, // 70% + } + defaultCalculateIndicatorWeight = map[IndicatorType]int64{ + consts.Usage15MinAvgKey: 30, //30% + consts.Usage1HourMaxKey: 30, //30% + consts.Usage1DayMaxKey: 40, //40% + } + defaultResourceToTargetMap = map[v1.ResourceName]int64{ + v1.ResourceCPU: 50, + v1.ResourceMemory: 70, + } +) + // SetDefaults_QoSAwareNodeResourcesFitArgs sets the default parameters for QoSAwareNodeResourcesFit plugin. func SetDefaults_QoSAwareNodeResourcesFitArgs(obj *QoSAwareNodeResourcesFitArgs) { if obj.ScoringStrategy == nil { @@ -107,3 +135,30 @@ func SetDefaults_NodeResourceTopologyArgs(obj *NodeResourceTopologyArgs) { obj.ResourcePluginPolicy = consts.ResourcePluginPolicyNameDynamic } } + +func SetDefaults_LoadAwareArgs(obj *LoadAwareArgs) { + if obj.FilterExpiredNodeMetrics == nil { + obj.FilterExpiredNodeMetrics = pointer.BoolPtr(true) + } + if obj.NodeMetricsExpiredSeconds == nil { + obj.NodeMetricsExpiredSeconds = pointer.Int64Ptr(defaultNodeMonitorExpiredSeconds) + } + if len(obj.ResourceToWeightMap) == 0 { + obj.ResourceToWeightMap = defaultResourceToWeightMap + } + if len(obj.ResourceToThresholdMap) == 0 { + obj.ResourceToThresholdMap = defaultResourceToThresholdMap + } + if len(obj.ResourceToScalingFactorMap) == 0 { + obj.ResourceToScalingFactorMap = defaultResourceToScalingFactorMap + } + if len(obj.CalculateIndicatorWeight) == 0 { + obj.CalculateIndicatorWeight = defaultCalculateIndicatorWeight + } + if len(obj.ResourceToTargetMap) == 0 { + obj.ResourceToTargetMap = defaultResourceToTargetMap + } + if obj.EnablePortrait == nil { + obj.EnablePortrait = pointer.Bool(false) + } +} diff --git a/pkg/apis/scheduling/config/v1beta3/register.go b/pkg/apis/scheduling/config/v1beta3/register.go index 9c360534..0adf01b5 100644 --- a/pkg/apis/scheduling/config/v1beta3/register.go +++ b/pkg/apis/scheduling/config/v1beta3/register.go @@ -35,6 +35,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { &QoSAwareNodeResourcesFitArgs{}, &QoSAwareNodeResourcesBalancedAllocationArgs{}, &NodeResourceTopologyArgs{}, + &LoadAwareArgs{}, ) return nil } diff --git a/pkg/apis/scheduling/config/v1beta3/types.go b/pkg/apis/scheduling/config/v1beta3/types.go index 70a12e75..d971704e 100644 --- a/pkg/apis/scheduling/config/v1beta3/types.go +++ b/pkg/apis/scheduling/config/v1beta3/types.go @@ -15,6 +15,7 @@ package v1beta3 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kube-scheduler/config/v1beta3" @@ -85,3 +86,37 @@ type NodeResourceTopologyArgs struct { // ResourcePluginPolicy are QRMPlugin resource policy to allocate topology resource for containers. ResourcePluginPolicy consts.ResourcePluginPolicyName `json:"resourcePluginPolicy,omitempty"` } + +// IndicatorType indicator participate in calculate score +type IndicatorType string + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// LoadAwareArgs holds arguments used to configure the LoadAwareScheduling plugin. +type LoadAwareArgs struct { + metav1.TypeMeta + + EnablePortrait *bool `json:"enablePortrait,omitempty"` + PodAnnotationLoadAwareEnable *string `json:"podAnnotationLoadAwareEnable,omitempty"` + // FilterExpiredNodeMetrics indicates whether to filter nodes where fails to update NPD. + FilterExpiredNodeMetrics *bool `json:"filterExpiredNodeMetrics,omitempty"` + // NodeMetricsExpiredSeconds indicates the NodeMetrics in NPD expiration in seconds. + // When NodeMetrics expired, the node is considered abnormal. + // default 5 minute + NodeMetricsExpiredSeconds *int64 `json:"NodeMetricsExpiredSeconds,omitempty"` + // ResourceToWeightMap contains resource name and weight. + ResourceToWeightMap map[corev1.ResourceName]int64 `json:"resourceToWeightMap,omitempty"` + // ResourceToThresholdMap contains resource name and threshold. Node can not be scheduled + // if usage of it is more than threshold. + ResourceToThresholdMap map[corev1.ResourceName]int64 `json:"resourceToThresholdMap,omitempty"` + // ResourceToScalingFactorMap contains resource name and scaling factor, which are used to estimate pod usage + // if usage of pod is not exists in node monitor. + ResourceToScalingFactorMap map[corev1.ResourceName]int64 `json:"resourceToScalingFactorMap,omitempty"` + // ResourceToTargetMap contains resource name and node usage target which are used in loadAware score + ResourceToTargetMap map[corev1.ResourceName]int64 `json:"resourceToTargetMap,omitempty"` + // CalculateIndicatorWeight indicates the participates in calculate indicator weight + // The default avg_15min 30, max_1hour 30, max_1day 40 + CalculateIndicatorWeight map[IndicatorType]int64 `json:"calculateIndicatorWeight,omitempty"` + + KubeConfigPath string `json:"kubeConfigPath,omitempty"` +} diff --git a/pkg/apis/scheduling/config/v1beta3/zz_generated.conversion.go b/pkg/apis/scheduling/config/v1beta3/zz_generated.conversion.go index af340494..a1ecc197 100644 --- a/pkg/apis/scheduling/config/v1beta3/zz_generated.conversion.go +++ b/pkg/apis/scheduling/config/v1beta3/zz_generated.conversion.go @@ -26,6 +26,7 @@ import ( config "github.com/kubewharf/katalyst-api/pkg/apis/scheduling/config" consts "github.com/kubewharf/katalyst-api/pkg/consts" + v1 "k8s.io/api/core/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" configv1beta3 "k8s.io/kube-scheduler/config/v1beta3" @@ -39,6 +40,16 @@ func init() { // RegisterConversions adds conversion functions to the given scheme. // Public to allow building arbitrary schemes. func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*LoadAwareArgs)(nil), (*config.LoadAwareArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_LoadAwareArgs_To_config_LoadAwareArgs(a.(*LoadAwareArgs), b.(*config.LoadAwareArgs), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.LoadAwareArgs)(nil), (*LoadAwareArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_LoadAwareArgs_To_v1beta3_LoadAwareArgs(a.(*config.LoadAwareArgs), b.(*LoadAwareArgs), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*NodeResourceTopologyArgs)(nil), (*config.NodeResourceTopologyArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_NodeResourceTopologyArgs_To_config_NodeResourceTopologyArgs(a.(*NodeResourceTopologyArgs), b.(*config.NodeResourceTopologyArgs), scope) }); err != nil { @@ -82,6 +93,44 @@ func RegisterConversions(s *runtime.Scheme) error { return nil } +func autoConvert_v1beta3_LoadAwareArgs_To_config_LoadAwareArgs(in *LoadAwareArgs, out *config.LoadAwareArgs, s conversion.Scope) error { + out.EnablePortrait = (*bool)(unsafe.Pointer(in.EnablePortrait)) + out.PodAnnotationLoadAwareEnable = (*string)(unsafe.Pointer(in.PodAnnotationLoadAwareEnable)) + out.FilterExpiredNodeMetrics = (*bool)(unsafe.Pointer(in.FilterExpiredNodeMetrics)) + out.NodeMetricsExpiredSeconds = (*int64)(unsafe.Pointer(in.NodeMetricsExpiredSeconds)) + out.ResourceToWeightMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToWeightMap)) + out.ResourceToThresholdMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToThresholdMap)) + out.ResourceToScalingFactorMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToScalingFactorMap)) + out.ResourceToTargetMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToTargetMap)) + out.CalculateIndicatorWeight = *(*map[config.IndicatorType]int64)(unsafe.Pointer(&in.CalculateIndicatorWeight)) + out.KubeConfigPath = in.KubeConfigPath + return nil +} + +// Convert_v1beta3_LoadAwareArgs_To_config_LoadAwareArgs is an autogenerated conversion function. +func Convert_v1beta3_LoadAwareArgs_To_config_LoadAwareArgs(in *LoadAwareArgs, out *config.LoadAwareArgs, s conversion.Scope) error { + return autoConvert_v1beta3_LoadAwareArgs_To_config_LoadAwareArgs(in, out, s) +} + +func autoConvert_config_LoadAwareArgs_To_v1beta3_LoadAwareArgs(in *config.LoadAwareArgs, out *LoadAwareArgs, s conversion.Scope) error { + out.EnablePortrait = (*bool)(unsafe.Pointer(in.EnablePortrait)) + out.PodAnnotationLoadAwareEnable = (*string)(unsafe.Pointer(in.PodAnnotationLoadAwareEnable)) + out.FilterExpiredNodeMetrics = (*bool)(unsafe.Pointer(in.FilterExpiredNodeMetrics)) + out.NodeMetricsExpiredSeconds = (*int64)(unsafe.Pointer(in.NodeMetricsExpiredSeconds)) + out.ResourceToWeightMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToWeightMap)) + out.ResourceToThresholdMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToThresholdMap)) + out.ResourceToScalingFactorMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToScalingFactorMap)) + out.ResourceToTargetMap = *(*map[v1.ResourceName]int64)(unsafe.Pointer(&in.ResourceToTargetMap)) + out.CalculateIndicatorWeight = *(*map[IndicatorType]int64)(unsafe.Pointer(&in.CalculateIndicatorWeight)) + out.KubeConfigPath = in.KubeConfigPath + return nil +} + +// Convert_config_LoadAwareArgs_To_v1beta3_LoadAwareArgs is an autogenerated conversion function. +func Convert_config_LoadAwareArgs_To_v1beta3_LoadAwareArgs(in *config.LoadAwareArgs, out *LoadAwareArgs, s conversion.Scope) error { + return autoConvert_config_LoadAwareArgs_To_v1beta3_LoadAwareArgs(in, out, s) +} + func autoConvert_v1beta3_NodeResourceTopologyArgs_To_config_NodeResourceTopologyArgs(in *NodeResourceTopologyArgs, out *config.NodeResourceTopologyArgs, s conversion.Scope) error { out.ScoringStrategy = (*config.ScoringStrategy)(unsafe.Pointer(in.ScoringStrategy)) out.AlignedResources = *(*[]string)(unsafe.Pointer(&in.AlignedResources)) diff --git a/pkg/apis/scheduling/config/v1beta3/zz_generated.deepcopy.go b/pkg/apis/scheduling/config/v1beta3/zz_generated.deepcopy.go index 286ccffc..77a33dbb 100644 --- a/pkg/apis/scheduling/config/v1beta3/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/config/v1beta3/zz_generated.deepcopy.go @@ -22,10 +22,91 @@ limitations under the License. package v1beta3 import ( + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" configv1beta3 "k8s.io/kube-scheduler/config/v1beta3" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadAwareArgs) DeepCopyInto(out *LoadAwareArgs) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EnablePortrait != nil { + in, out := &in.EnablePortrait, &out.EnablePortrait + *out = new(bool) + **out = **in + } + if in.PodAnnotationLoadAwareEnable != nil { + in, out := &in.PodAnnotationLoadAwareEnable, &out.PodAnnotationLoadAwareEnable + *out = new(string) + **out = **in + } + if in.FilterExpiredNodeMetrics != nil { + in, out := &in.FilterExpiredNodeMetrics, &out.FilterExpiredNodeMetrics + *out = new(bool) + **out = **in + } + if in.NodeMetricsExpiredSeconds != nil { + in, out := &in.NodeMetricsExpiredSeconds, &out.NodeMetricsExpiredSeconds + *out = new(int64) + **out = **in + } + if in.ResourceToWeightMap != nil { + in, out := &in.ResourceToWeightMap, &out.ResourceToWeightMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ResourceToThresholdMap != nil { + in, out := &in.ResourceToThresholdMap, &out.ResourceToThresholdMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ResourceToScalingFactorMap != nil { + in, out := &in.ResourceToScalingFactorMap, &out.ResourceToScalingFactorMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ResourceToTargetMap != nil { + in, out := &in.ResourceToTargetMap, &out.ResourceToTargetMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.CalculateIndicatorWeight != nil { + in, out := &in.CalculateIndicatorWeight, &out.CalculateIndicatorWeight + *out = make(map[IndicatorType]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadAwareArgs. +func (in *LoadAwareArgs) DeepCopy() *LoadAwareArgs { + if in == nil { + return nil + } + out := new(LoadAwareArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadAwareArgs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeResourceTopologyArgs) DeepCopyInto(out *NodeResourceTopologyArgs) { *out = *in diff --git a/pkg/apis/scheduling/config/v1beta3/zz_generated.defaults.go b/pkg/apis/scheduling/config/v1beta3/zz_generated.defaults.go index 8c755f91..029fe4ee 100644 --- a/pkg/apis/scheduling/config/v1beta3/zz_generated.defaults.go +++ b/pkg/apis/scheduling/config/v1beta3/zz_generated.defaults.go @@ -29,6 +29,7 @@ import ( // Public to allow building arbitrary schemes. // All generated defaulters are covering - they call all nested defaulters. func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&LoadAwareArgs{}, func(obj interface{}) { SetObjectDefaults_LoadAwareArgs(obj.(*LoadAwareArgs)) }) scheme.AddTypeDefaultingFunc(&NodeResourceTopologyArgs{}, func(obj interface{}) { SetObjectDefaults_NodeResourceTopologyArgs(obj.(*NodeResourceTopologyArgs)) }) scheme.AddTypeDefaultingFunc(&QoSAwareNodeResourcesBalancedAllocationArgs{}, func(obj interface{}) { SetObjectDefaults_QoSAwareNodeResourcesBalancedAllocationArgs(obj.(*QoSAwareNodeResourcesBalancedAllocationArgs)) @@ -39,6 +40,10 @@ func RegisterDefaults(scheme *runtime.Scheme) error { return nil } +func SetObjectDefaults_LoadAwareArgs(in *LoadAwareArgs) { + SetDefaults_LoadAwareArgs(in) +} + func SetObjectDefaults_NodeResourceTopologyArgs(in *NodeResourceTopologyArgs) { SetDefaults_NodeResourceTopologyArgs(in) } diff --git a/pkg/apis/scheduling/config/validation/validation_pluginargs.go b/pkg/apis/scheduling/config/validation/validation_pluginargs.go index 1c1fd157..2ac445c3 100644 --- a/pkg/apis/scheduling/config/validation/validation_pluginargs.go +++ b/pkg/apis/scheduling/config/validation/validation_pluginargs.go @@ -17,6 +17,7 @@ package validation import ( "fmt" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -156,3 +157,91 @@ func validateResourcePolicy(resourcePolicy consts.ResourcePluginPolicyName, path } return nil } + +// ValidateLoadAwareSchedulingArgs validates that LoadAwareArgs are correct. +func ValidateLoadAwareSchedulingArgs(args *config.LoadAwareArgs) error { + if args.NodeMetricsExpiredSeconds != nil && *args.NodeMetricsExpiredSeconds <= 0 { + return fmt.Errorf("NodeMonitorExpiredSeconds err, NodeMonitorExpiredSeconds should be a positive value") + } + if err := validateResourceWeights(args.ResourceToWeightMap); err != nil { + return fmt.Errorf("ResourceWeights err, %v", err) + } + if err := validateResourceThresholds(args.ResourceToThresholdMap); err != nil { + return fmt.Errorf("UsageThresholds err, %v", err) + } + if err := validateEstimatedResourceThresholds(args.ResourceToScalingFactorMap); err != nil { + return fmt.Errorf("EstimatedScalingFactors err, %v", err) + } + for resourceName := range args.ResourceToWeightMap { + if _, ok := args.ResourceToScalingFactorMap[resourceName]; !ok { + return fmt.Errorf("LoadAwareValidating err, resourceName %v in ResourceWeights, but not find in EstimatedScalingFactors", resourceName) + } + } + if err := validateCalculateIndicatorWeight(args.CalculateIndicatorWeight); err != nil { + return fmt.Errorf("CalculateIndicatorWeight err, %v", err) + } + if err := validateResourceTargets(args.ResourceToTargetMap); err != nil { + return fmt.Errorf("UsageTarget err, %v", err) + } + return nil +} +func validateResourceWeights(resources map[v1.ResourceName]int64) error { + for resourceName, weight := range resources { + if weight < 0 { + return fmt.Errorf("resource Weight of %v should be a positive value, got %v", resourceName, weight) + } + if weight > 100 { + return fmt.Errorf("resource Weight of %v should be less than 100, got %v", resourceName, weight) + } + } + return nil +} + +func validateResourceThresholds(thresholds map[v1.ResourceName]int64) error { + for resourceName, thresholdPercent := range thresholds { + if thresholdPercent < 0 { + return fmt.Errorf("resource Threshold of %v should be a positive value, got %v", resourceName, thresholdPercent) + } + if thresholdPercent > 100 { + return fmt.Errorf("resource Threshold of %v should be less than 100, got %v", resourceName, thresholdPercent) + } + } + return nil +} + +func validateEstimatedResourceThresholds(thresholds map[v1.ResourceName]int64) error { + for resourceName, thresholdPercent := range thresholds { + if thresholdPercent < 0 { + return fmt.Errorf("estimated resource Threshold of %v should be a positive value, got %v", resourceName, thresholdPercent) + } + if thresholdPercent > 100 { + return fmt.Errorf("estimated resource Threshold of %v should be less than 100, got %v", resourceName, thresholdPercent) + } + } + return nil +} + +func validateCalculateIndicatorWeight(calculateIndicatorWeight map[config.IndicatorType]int64) error { + for indicator, weight := range calculateIndicatorWeight { + if weight < 0 { + return fmt.Errorf("calculate Indicator weight of %v should be a positive value, got %v", indicator, weight) + } + if weight > 100 { + return fmt.Errorf("calculate Indicator weight of %v should be less than 100, got %v", indicator, weight) + } + } + return nil +} + +func validateResourceTargets(targets map[v1.ResourceName]int64) error { + for resourceName, target := range targets { + if target < 0 { + return fmt.Errorf("resource target of %v should be a positive value, got %v", resourceName, target) + } + + if target > 100 { + return fmt.Errorf("resource target of %v should be less than 100, got %v", resourceName, target) + } + } + return nil +} diff --git a/pkg/apis/scheduling/config/zz_generated.deepcopy.go b/pkg/apis/scheduling/config/zz_generated.deepcopy.go index 46d4a2df..3763ad7e 100644 --- a/pkg/apis/scheduling/config/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/config/zz_generated.deepcopy.go @@ -22,10 +22,91 @@ limitations under the License. package config import ( + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" apisconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadAwareArgs) DeepCopyInto(out *LoadAwareArgs) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EnablePortrait != nil { + in, out := &in.EnablePortrait, &out.EnablePortrait + *out = new(bool) + **out = **in + } + if in.PodAnnotationLoadAwareEnable != nil { + in, out := &in.PodAnnotationLoadAwareEnable, &out.PodAnnotationLoadAwareEnable + *out = new(string) + **out = **in + } + if in.FilterExpiredNodeMetrics != nil { + in, out := &in.FilterExpiredNodeMetrics, &out.FilterExpiredNodeMetrics + *out = new(bool) + **out = **in + } + if in.NodeMetricsExpiredSeconds != nil { + in, out := &in.NodeMetricsExpiredSeconds, &out.NodeMetricsExpiredSeconds + *out = new(int64) + **out = **in + } + if in.ResourceToWeightMap != nil { + in, out := &in.ResourceToWeightMap, &out.ResourceToWeightMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ResourceToThresholdMap != nil { + in, out := &in.ResourceToThresholdMap, &out.ResourceToThresholdMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ResourceToScalingFactorMap != nil { + in, out := &in.ResourceToScalingFactorMap, &out.ResourceToScalingFactorMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ResourceToTargetMap != nil { + in, out := &in.ResourceToTargetMap, &out.ResourceToTargetMap + *out = make(map[v1.ResourceName]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.CalculateIndicatorWeight != nil { + in, out := &in.CalculateIndicatorWeight, &out.CalculateIndicatorWeight + *out = make(map[IndicatorType]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadAwareArgs. +func (in *LoadAwareArgs) DeepCopy() *LoadAwareArgs { + if in == nil { + return nil + } + out := new(LoadAwareArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LoadAwareArgs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeResourceTopologyArgs) DeepCopyInto(out *NodeResourceTopologyArgs) { *out = *in diff --git a/pkg/consts/loadaware.go b/pkg/consts/loadaware.go new file mode 100644 index 00000000..27991d67 --- /dev/null +++ b/pkg/consts/loadaware.go @@ -0,0 +1,39 @@ +// Copyright 2022 The Katalyst Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consts + +const ( + PodAnnotationLoadAwareEnableTrue = "true" + PodAnnotationLoadAwareEnableFalse = "false" + + DefaultPodAnnotationEnableLoadAware = "katalyst.kubewharf.io/enable_load_aware" +) + +const ( + // Usage5MinAvgKey ... + Usage5MinAvgKey = "avg_5min" + // Usage15MinAvgKey ... + Usage15MinAvgKey = "avg_15min" + // Usage1HourMaxKey ... + Usage1HourMaxKey = "max_1hour" + // Usage1DayMaxKey ... + Usage1DayMaxKey = "max_1day" + // Annotation15MinAvgKey ... + Annotation15MinAvgKey = "avg_15min_metadata" + // Annotation1HourMaxKey ... + Annotation1HourMaxKey = "max_1hour_metadata" + // Annotation1DayMaxKey ... + Annotation1DayMaxKey = "max_1day_metadata" +)