feat(scheduler): add loadaware scheduler plugin args

go.mod

@@ -14,6 +14,7 @@ require (
 	k8s.io/kubelet v0.24.6
 	k8s.io/kubernetes v1.24.6
 	k8s.io/metrics v0.24.6
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 	sigs.k8s.io/yaml v1.2.0
 )
 
@@ -59,7 +60,6 @@ require (
 	k8s.io/component-base v0.24.6 // indirect
 	k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
 	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )

pkg/apis/scheduling/config/register.go

@@ -35,6 +35,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&QoSAwareNodeResourcesFitArgs{},
 		&QoSAwareNodeResourcesBalancedAllocationArgs{},
 		&NodeResourceTopologyArgs{},
+		&LoadAwareArgs{},
 	)
 	return nil
 }

pkg/apis/scheduling/config/types.go

@@ -15,6 +15,7 @@
 package config
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 
@@ -85,3 +86,37 @@ type ScoringStrategy struct {
 	// Arguments specific to RequestedToCapacityRatio strategy.
 	ReclaimedRequestedToCapacityRatio *kubeschedulerconfig.RequestedToCapacityRatioParam `json:"reclaimedRequestedToCapacityRatio,omitempty"`
 }
+
+// IndicatorType indicator participate in calculate score
+type IndicatorType string
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// LoadAwareArgs holds arguments used to configure the LoadAwareScheduling plugin.
+type LoadAwareArgs struct {
+	metav1.TypeMeta
+
+	EnablePortrait               *bool   `json:"enablePortrait,omitempty"`
+	PodAnnotationLoadAwareEnable *string `json:"podAnnotationLoadAwareEnable,omitempty"`
+	// FilterExpiredNodeMetrics indicates whether to filter nodes where  fails to update NPD.
+	FilterExpiredNodeMetrics *bool `json:"filterExpiredNodeMetrics,omitempty"`
+	// NodeMetricsExpiredSeconds indicates the NodeMetrics in NPD expiration in seconds.
+	// When NodeMetrics expired, the node is considered abnormal.
+	// default 5 minute
+	NodeMetricsExpiredSeconds *int64 `json:"NodeMetricsExpiredSeconds,omitempty"`
+	// ResourceToWeightMap contains resource name and weight.
+	ResourceToWeightMap map[corev1.ResourceName]int64 `json:"resourceToWeightMap,omitempty"`
+	// ResourceToThresholdMap contains resource name and threshold. Node can not be scheduled
+	// if usage of it is more than threshold.
+	ResourceToThresholdMap map[corev1.ResourceName]int64 `json:"resourceToThresholdMap,omitempty"`
+	// ResourceToScalingFactorMap contains resource name and scaling factor, which are used to estimate pod usage
+	// if usage of pod is not exists in node monitor.
+	ResourceToScalingFactorMap map[corev1.ResourceName]int64 `json:"resourceToScalingFactorMap,omitempty"`
+	// ResourceToTargetMap contains resource name and node usage target which are used in loadAware score
+	ResourceToTargetMap map[corev1.ResourceName]int64 `json:"resourceToTargetMap,omitempty"`
+	// CalculateIndicatorWeight indicates the participates in calculate indicator weight
+	// The default avg_15min 30, max_1hour 30, max_1day 40
+	CalculateIndicatorWeight map[IndicatorType]int64 `json:"calculateIndicatorWeight,omitempty"`
+
+	KubeConfigPath string `json:"kubeConfigPath,omitempty"`
+}

pkg/apis/scheduling/config/v1beta3/defaults.go

@@ -19,6 +19,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/kube-scheduler/config/v1beta3"
+	"k8s.io/utils/pointer"
 
 	"github.com/kubewharf/katalyst-api/pkg/consts"
 )
@@ -35,6 +36,33 @@ var defaultReclaimedResourceSpec = []v1beta3.ResourceSpec{
 
 var defaultAlignedResourceSpec = []string{v1.ResourceCPU.String(), v1.ResourceMemory.String()}
 
+var (
+	defaultNodeMonitorExpiredSeconds int64 = 180
+	defaultResourceToWeightMap             = map[v1.ResourceName]int64{
+		v1.ResourceCPU:    1,
+		v1.ResourceMemory: 1,
+	}
+
+	defaultResourceToThresholdMap = map[v1.ResourceName]int64{
+		v1.ResourceCPU:    70, // 70%
+		v1.ResourceMemory: 95, // 95%
+	}
+
+	defaultResourceToScalingFactorMap = map[v1.ResourceName]int64{
+		v1.ResourceCPU:    85, // 85%
+		v1.ResourceMemory: 70, // 70%
+	}
+	defaultCalculateIndicatorWeight = map[IndicatorType]int64{
+		consts.Usage15MinAvgKey: 30, //30%
+		consts.Usage1HourMaxKey: 30, //30%
+		consts.Usage1DayMaxKey:  40, //40%
+	}
+	defaultResourceToTargetMap = map[v1.ResourceName]int64{
+		v1.ResourceCPU:    50,
+		v1.ResourceMemory: 70,
+	}
+)
+
 // SetDefaults_QoSAwareNodeResourcesFitArgs sets the default parameters for QoSAwareNodeResourcesFit plugin.
 func SetDefaults_QoSAwareNodeResourcesFitArgs(obj *QoSAwareNodeResourcesFitArgs) {
 	if obj.ScoringStrategy == nil {
@@ -107,3 +135,30 @@ func SetDefaults_NodeResourceTopologyArgs(obj *NodeResourceTopologyArgs) {
 		obj.ResourcePluginPolicy = consts.ResourcePluginPolicyNameDynamic
 	}
 }
+
+func SetDefaults_LoadAwareArgs(obj *LoadAwareArgs) {
+	if obj.FilterExpiredNodeMetrics == nil {
+		obj.FilterExpiredNodeMetrics = pointer.BoolPtr(true)
+	}
+	if obj.NodeMetricsExpiredSeconds == nil {
+		obj.NodeMetricsExpiredSeconds = pointer.Int64Ptr(defaultNodeMonitorExpiredSeconds)
+	}
+	if len(obj.ResourceToWeightMap) == 0 {
+		obj.ResourceToWeightMap = defaultResourceToWeightMap
+	}
+	if len(obj.ResourceToThresholdMap) == 0 {
+		obj.ResourceToThresholdMap = defaultResourceToThresholdMap
+	}
+	if len(obj.ResourceToScalingFactorMap) == 0 {
+		obj.ResourceToScalingFactorMap = defaultResourceToScalingFactorMap
+	}
+	if len(obj.CalculateIndicatorWeight) == 0 {
+		obj.CalculateIndicatorWeight = defaultCalculateIndicatorWeight
+	}
+	if len(obj.ResourceToTargetMap) == 0 {
+		obj.ResourceToTargetMap = defaultResourceToTargetMap
+	}
+	if obj.EnablePortrait == nil {
+		obj.EnablePortrait = pointer.Bool(false)
+	}
+}

pkg/apis/scheduling/config/v1beta3/register.go

@@ -35,6 +35,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&QoSAwareNodeResourcesFitArgs{},
 		&QoSAwareNodeResourcesBalancedAllocationArgs{},
 		&NodeResourceTopologyArgs{},
+		&LoadAwareArgs{},
 	)
 	return nil
 }

pkg/apis/scheduling/config/v1beta3/types.go

@@ -15,6 +15,7 @@
 package v1beta3
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kube-scheduler/config/v1beta3"
 
@@ -85,3 +86,37 @@ type NodeResourceTopologyArgs struct {
 	// ResourcePluginPolicy are QRMPlugin resource policy to allocate topology resource for containers.
 	ResourcePluginPolicy consts.ResourcePluginPolicyName `json:"resourcePluginPolicy,omitempty"`
 }
+
+// IndicatorType indicator participate in calculate score
+type IndicatorType string
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// LoadAwareArgs holds arguments used to configure the LoadAwareScheduling plugin.
+type LoadAwareArgs struct {
+	metav1.TypeMeta
+
+	EnablePortrait               *bool   `json:"enablePortrait,omitempty"`
+	PodAnnotationLoadAwareEnable *string `json:"podAnnotationLoadAwareEnable,omitempty"`
+	// FilterExpiredNodeMetrics indicates whether to filter nodes where  fails to update NPD.
+	FilterExpiredNodeMetrics *bool `json:"filterExpiredNodeMetrics,omitempty"`
+	// NodeMetricsExpiredSeconds indicates the NodeMetrics in NPD expiration in seconds.
+	// When NodeMetrics expired, the node is considered abnormal.
+	// default 5 minute
+	NodeMetricsExpiredSeconds *int64 `json:"NodeMetricsExpiredSeconds,omitempty"`
+	// ResourceToWeightMap contains resource name and weight.
+	ResourceToWeightMap map[corev1.ResourceName]int64 `json:"resourceToWeightMap,omitempty"`
+	// ResourceToThresholdMap contains resource name and threshold. Node can not be scheduled
+	// if usage of it is more than threshold.
+	ResourceToThresholdMap map[corev1.ResourceName]int64 `json:"resourceToThresholdMap,omitempty"`
+	// ResourceToScalingFactorMap contains resource name and scaling factor, which are used to estimate pod usage
+	// if usage of pod is not exists in node monitor.
+	ResourceToScalingFactorMap map[corev1.ResourceName]int64 `json:"resourceToScalingFactorMap,omitempty"`
+	// ResourceToTargetMap contains resource name and node usage target which are used in loadAware score
+	ResourceToTargetMap map[corev1.ResourceName]int64 `json:"resourceToTargetMap,omitempty"`
+	// CalculateIndicatorWeight indicates the participates in calculate indicator weight
+	// The default avg_15min 30, max_1hour 30, max_1day 40
+	CalculateIndicatorWeight map[IndicatorType]int64 `json:"calculateIndicatorWeight,omitempty"`
+
+	KubeConfigPath string `json:"kubeConfigPath,omitempty"`
+}