diff --git a/pkg/agent/resourcemanager/fetcher/kubelet/kubeletplugin_test.go b/pkg/agent/resourcemanager/fetcher/kubelet/kubeletplugin_test.go index eb16faa56..bb6e3764a 100644 --- a/pkg/agent/resourcemanager/fetcher/kubelet/kubeletplugin_test.go +++ b/pkg/agent/resourcemanager/fetcher/kubelet/kubeletplugin_test.go @@ -31,7 +31,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" podresv1 "k8s.io/kubelet/pkg/apis/podresources/v1" apisconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" @@ -49,6 +48,7 @@ import ( "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" "github.com/kubewharf/katalyst-core/pkg/metrics" "github.com/kubewharf/katalyst-core/pkg/util/machine" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) type fakePodResourcesServer struct { @@ -89,7 +89,7 @@ func generateTestConfiguration(t *testing.T, dir string) *config.Configuration { } func generateTestMetaServer(podList ...*v1.Pod) *metaserver.MetaServer { - fakeKubeletConfig := kubeletconfigv1beta1.KubeletConfiguration{ + fakeKubeletConfig := native.KubeletConfiguration{ TopologyManagerPolicy: apisconfig.SingleNumaNodeTopologyManagerPolicy, TopologyManagerScope: apisconfig.ContainerTopologyManagerScope, } diff --git a/pkg/agent/resourcemanager/fetcher/kubelet/topology/topology_adapter_test.go b/pkg/agent/resourcemanager/fetcher/kubelet/topology/topology_adapter_test.go index 749760ebb..c70de2e52 100644 --- a/pkg/agent/resourcemanager/fetcher/kubelet/topology/topology_adapter_test.go +++ b/pkg/agent/resourcemanager/fetcher/kubelet/topology/topology_adapter_test.go @@ -33,7 +33,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" podresv1 "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" @@ -51,6 +50,7 @@ import ( "github.com/kubewharf/katalyst-core/pkg/util" "github.com/kubewharf/katalyst-core/pkg/util/kubelet/podresources" "github.com/kubewharf/katalyst-core/pkg/util/machine" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) type fakePodResourcesServer struct { @@ -2913,7 +2913,7 @@ func Test_podResourcesServerTopologyAdapterImpl_GetTopologyZones(t *testing.T) { func Test_podResourcesServerTopologyAdapterImpl_GetTopologyPolicy(t *testing.T) { t.Parallel() - fakeKubeletConfig := kubeletconfigv1beta1.KubeletConfiguration{ + fakeKubeletConfig := native.KubeletConfiguration{ TopologyManagerPolicy: config.SingleNumaNodeTopologyManagerPolicy, TopologyManagerScope: config.ContainerTopologyManagerScope, } diff --git a/pkg/agent/sysadvisor/plugin/overcommitmentaware/realtime/realtime_test.go b/pkg/agent/sysadvisor/plugin/overcommitmentaware/realtime/realtime_test.go index 94040c8cd..85edcef21 100644 --- a/pkg/agent/sysadvisor/plugin/overcommitmentaware/realtime/realtime_test.go +++ b/pkg/agent/sysadvisor/plugin/overcommitmentaware/realtime/realtime_test.go @@ -23,8 +23,6 @@ import ( "testing" "time" - consts2 "github.com/kubewharf/katalyst-api/pkg/consts" - info "github.com/google/cadvisor/info/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -34,9 +32,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" - "k8s.io/kubelet/config/v1beta1" internalfake "github.com/kubewharf/katalyst-api/pkg/client/clientset/versioned/fake" + consts2 "github.com/kubewharf/katalyst-api/pkg/consts" "github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/options" "github.com/kubewharf/katalyst-core/pkg/client" "github.com/kubewharf/katalyst-core/pkg/config" @@ -47,6 +45,7 @@ import ( "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" "github.com/kubewharf/katalyst-core/pkg/metrics" metric2 "github.com/kubewharf/katalyst-core/pkg/util/metric" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) func TestUpdate(t *testing.T) { @@ -437,7 +436,7 @@ func generateTestMetaServer(t *testing.T, conf *config.Configuration, podList [] meta.MetricsFetcher = fakeMetricsFetcher - meta.KubeletConfigFetcher = kubeletconfig.NewFakeKubeletConfigFetcher(v1beta1.KubeletConfiguration{}) + meta.KubeletConfigFetcher = kubeletconfig.NewFakeKubeletConfigFetcher(native.KubeletConfiguration{}) meta.MachineInfo = &info.MachineInfo{ NumCores: 16, diff --git a/pkg/agent/sysadvisor/plugin/overcommitmentaware/reporter/reporter_test.go b/pkg/agent/sysadvisor/plugin/overcommitmentaware/reporter/reporter_test.go index 2c6372e84..3d6b3e105 100644 --- a/pkg/agent/sysadvisor/plugin/overcommitmentaware/reporter/reporter_test.go +++ b/pkg/agent/sysadvisor/plugin/overcommitmentaware/reporter/reporter_test.go @@ -26,7 +26,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" @@ -35,12 +34,13 @@ import ( "github.com/kubewharf/katalyst-core/pkg/metaserver/agent" "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/kubeletconfig" "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) func TestGetReportContent(t *testing.T) { t.Parallel() - fakeKubeletConfig := kubeletconfigv1beta1.KubeletConfiguration{ + fakeKubeletConfig := native.KubeletConfiguration{ FeatureGates: map[string]bool{ string(features.CPUManager): true, string(features.MemoryManager): false, diff --git a/pkg/metaserver/agent/agent_test.go b/pkg/metaserver/agent/agent_test.go index 7674a0851..5393d26b1 100644 --- a/pkg/metaserver/agent/agent_test.go +++ b/pkg/metaserver/agent/agent_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/kubelet/apis/config" "github.com/kubewharf/katalyst-api/pkg/apis/node/v1alpha1" @@ -45,6 +44,7 @@ import ( "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/node" "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" "github.com/kubewharf/katalyst-core/pkg/metrics" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) func constructNodeInterface(name string) corev1.NodeInterface { @@ -169,7 +169,7 @@ func TestFetcher(t *testing.T) { } s := &ObjectFetcherTest{obj: obj} - fakeKubeletConfig := kubeletconfigv1beta1.KubeletConfiguration{ + fakeKubeletConfig := native.KubeletConfiguration{ TopologyManagerPolicy: config.SingleNumaNodeTopologyManagerPolicy, TopologyManagerScope: config.ContainerTopologyManagerScope, } diff --git a/pkg/metaserver/agent/kubeletconfig/fake_kubeletconfig.go b/pkg/metaserver/agent/kubeletconfig/fake_kubeletconfig.go index 7a5a6fc05..539ade5d7 100644 --- a/pkg/metaserver/agent/kubeletconfig/fake_kubeletconfig.go +++ b/pkg/metaserver/agent/kubeletconfig/fake_kubeletconfig.go @@ -19,11 +19,11 @@ package kubeletconfig import ( "context" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) // NewFakeKubeletConfigFetcher returns a fakeKubeletConfigFetcherImpl. -func NewFakeKubeletConfigFetcher(kubeletConfig kubeletconfigv1beta1.KubeletConfiguration) KubeletConfigFetcher { +func NewFakeKubeletConfigFetcher(kubeletConfig native.KubeletConfiguration) KubeletConfigFetcher { return &fakeKubeletConfigFetcherImpl{ kubeletConfig: kubeletConfig, } @@ -31,10 +31,10 @@ func NewFakeKubeletConfigFetcher(kubeletConfig kubeletconfigv1beta1.KubeletConfi // fakeKubeletConfigFetcherImpl returns a fake kubelet config. type fakeKubeletConfigFetcherImpl struct { - kubeletConfig kubeletconfigv1beta1.KubeletConfiguration + kubeletConfig native.KubeletConfiguration } // GetKubeletConfig returns a fake kubelet config. -func (f *fakeKubeletConfigFetcherImpl) GetKubeletConfig(_ context.Context) (*kubeletconfigv1beta1.KubeletConfiguration, error) { +func (f *fakeKubeletConfigFetcherImpl) GetKubeletConfig(_ context.Context) (*native.KubeletConfiguration, error) { return &f.kubeletConfig, nil } diff --git a/pkg/metaserver/agent/kubeletconfig/kubeletconfig.go b/pkg/metaserver/agent/kubeletconfig/kubeletconfig.go index c73549a5d..fb8bb4c1d 100644 --- a/pkg/metaserver/agent/kubeletconfig/kubeletconfig.go +++ b/pkg/metaserver/agent/kubeletconfig/kubeletconfig.go @@ -20,8 +20,6 @@ import ( "context" "fmt" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - "github.com/kubewharf/katalyst-core/pkg/config/agent/global" "github.com/kubewharf/katalyst-core/pkg/metrics" "github.com/kubewharf/katalyst-core/pkg/util/native" @@ -30,7 +28,7 @@ import ( // KubeletConfigFetcher is used to get the configuration of kubelet. type KubeletConfigFetcher interface { // GetKubeletConfig returns the configuration of kubelet. - GetKubeletConfig(ctx context.Context) (*kubeletconfigv1beta1.KubeletConfiguration, error) + GetKubeletConfig(ctx context.Context) (*native.KubeletConfiguration, error) } // NewKubeletConfigFetcher returns a KubeletConfigFetcher @@ -48,13 +46,13 @@ type kubeletConfigFetcherImpl struct { } // GetKubeletConfig gets kubelet config from kubelet 10250/configz api -func (k *kubeletConfigFetcherImpl) GetKubeletConfig(ctx context.Context) (*kubeletconfigv1beta1.KubeletConfiguration, error) { +func (k *kubeletConfigFetcherImpl) GetKubeletConfig(ctx context.Context) (*native.KubeletConfiguration, error) { if !k.baseConf.KubeletSecurePortEnabled { return nil, fmt.Errorf("it is not enabled to get contents from kubelet secure port") } type configzWrapper struct { - ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + ComponentConfig native.KubeletConfiguration `json:"kubeletconfig"` } configz := configzWrapper{} diff --git a/pkg/util/kubelet/config/kubeletconfig.go b/pkg/util/kubelet/config/kubeletconfig.go index cc0db0f2b..970984c5a 100644 --- a/pkg/util/kubelet/config/kubeletconfig.go +++ b/pkg/util/kubelet/config/kubeletconfig.go @@ -21,14 +21,14 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/klog/v2" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/features" "github.com/kubewharf/katalyst-api/pkg/consts" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) // CheckFeatureGateEnable returns true if all the given features are enabled -func CheckFeatureGateEnable(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, features ...string) (bool, error) { +func CheckFeatureGateEnable(kubeletConfig *native.KubeletConfiguration, features ...string) (bool, error) { if kubeletConfig == nil { return false, fmt.Errorf("nil KubeletConfiguration") } @@ -41,7 +41,7 @@ func CheckFeatureGateEnable(kubeletConfig *kubeletconfigv1beta1.KubeletConfigura } // GetReservedQuantity the quantity for reserved resources defined in KubeletConfiguration -func GetReservedQuantity(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, resourceName string) (resource.Quantity, bool, error) { +func GetReservedQuantity(kubeletConfig *native.KubeletConfiguration, resourceName string) (resource.Quantity, bool, error) { if kubeletConfig == nil { return resource.MustParse("0"), false, fmt.Errorf("nil KubeletConfiguration") } @@ -73,7 +73,7 @@ func GetReservedQuantity(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguratio // GetInTreeProviderPolicies returns a map containing the policy for in-tree // topology-hint-provider, i.e. cpu-manager && memory-manager -func GetInTreeProviderPolicies(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) (map[string]string, error) { +func GetInTreeProviderPolicies(kubeletConfig *native.KubeletConfiguration) (map[string]string, error) { if kubeletConfig == nil { return map[string]string{}, fmt.Errorf("nil KubeletConfiguration") } diff --git a/pkg/util/kubelet/config/kubeletconfig_test.go b/pkg/util/kubelet/config/kubeletconfig_test.go index afb47f738..cff82a7c2 100644 --- a/pkg/util/kubelet/config/kubeletconfig_test.go +++ b/pkg/util/kubelet/config/kubeletconfig_test.go @@ -22,10 +22,10 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/features" "github.com/kubewharf/katalyst-api/pkg/consts" + "github.com/kubewharf/katalyst-core/pkg/util/native" ) func TestCheckFeatureGateEnable(t *testing.T) { @@ -33,7 +33,7 @@ func TestCheckFeatureGateEnable(t *testing.T) { tests := []struct { name string - conf *kubeletconfigv1beta1.KubeletConfiguration + conf *native.KubeletConfiguration features []string enabled bool err error @@ -44,7 +44,7 @@ func TestCheckFeatureGateEnable(t *testing.T) { }, { name: "partial enabled", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ FeatureGates: map[string]bool{ "a": true, }, @@ -55,7 +55,7 @@ func TestCheckFeatureGateEnable(t *testing.T) { }, { name: "total enabled", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ FeatureGates: map[string]bool{ "a": true, "b": true, @@ -88,7 +88,7 @@ func TestGetReservedQuantity(t *testing.T) { tests := []struct { name string - conf *kubeletconfigv1beta1.KubeletConfiguration + conf *native.KubeletConfiguration resourceName string resourceQuantity resource.Quantity valid bool @@ -100,7 +100,7 @@ func TestGetReservedQuantity(t *testing.T) { }, { name: "resource both exists", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ KubeReserved: map[string]string{ "cpu": "1024m", }, @@ -115,7 +115,7 @@ func TestGetReservedQuantity(t *testing.T) { }, { name: "resource only-one exists", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ KubeReserved: map[string]string{ "cpu": "1024m", }, @@ -127,7 +127,7 @@ func TestGetReservedQuantity(t *testing.T) { }, { name: "resource not exists", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ KubeReserved: map[string]string{ "cpu": "1024m", }, @@ -162,7 +162,7 @@ func TestGetInTreeProviderPolicies(t *testing.T) { tests := []struct { name string - conf *kubeletconfigv1beta1.KubeletConfiguration + conf *native.KubeletConfiguration res map[string]string err error }{ @@ -172,7 +172,7 @@ func TestGetInTreeProviderPolicies(t *testing.T) { }, { name: "cpu only", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ FeatureGates: map[string]bool{ string(features.CPUManager): true, }, @@ -186,7 +186,7 @@ func TestGetInTreeProviderPolicies(t *testing.T) { }, { name: "all policies", - conf: &kubeletconfigv1beta1.KubeletConfiguration{ + conf: &native.KubeletConfiguration{ FeatureGates: map[string]bool{ string(features.CPUManager): true, string(features.MemoryManager): true, diff --git a/pkg/util/native/kubelet.go b/pkg/util/native/kubelet.go index 73693733b..531ca84e9 100644 --- a/pkg/util/native/kubelet.go +++ b/pkg/util/native/kubelet.go @@ -32,6 +32,83 @@ const ( defaultTimeout = time.Second * 10 ) +// KubeletConfiguration contains the configuration for the Kubelet +// This struct is a simplification of the definition in the kubelet api repo, holding only the fields used by katalyst. +type KubeletConfiguration struct { + // cpuManagerPolicy is the name of the policy to use. + // Requires the CPUManager feature gate to be enabled. + // Default: "None" + // +optional + CPUManagerPolicy string `json:"cpuManagerPolicy,omitempty"` + // memoryManagerPolicy is the name of the policy to use by memory manager. + // Requires the MemoryManager feature gate to be enabled. + // Default: "none" + // +optional + MemoryManagerPolicy string `json:"memoryManagerPolicy,omitempty"` + // topologyManagerPolicy is the name of the topology manager policy to use. + // Valid values include: + // + // - `restricted`: kubelet only allows pods with optimal NUMA node alignment for + // requested resources; + // - `best-effort`: kubelet will favor pods with NUMA alignment of CPU and device + // resources; + // - `none`: kubelet has no knowledge of NUMA alignment of a pod's CPU and device resources. + // - `single-numa-node`: kubelet only allows pods with a single NUMA alignment + // of CPU and device resources. + // + // Policies other than "none" require the TopologyManager feature gate to be enabled. + // Default: "none" + // +optional + TopologyManagerPolicy string `json:"topologyManagerPolicy,omitempty"` + // topologyManagerScope represents the scope of topology hint generation + // that topology manager requests and hint providers generate. Valid values include: + // + // - `container`: topology policy is applied on a per-container basis. + // - `pod`: topology policy is applied on a per-pod basis. + // + // "pod" scope requires the TopologyManager feature gate to be enabled. + // Default: "container" + // +optional + TopologyManagerScope string `json:"topologyManagerScope,omitempty"` + // featureGates is a map of feature names to bools that enable or disable experimental + // features. This field modifies piecemeal the built-in default values from + // "k8s.io/kubernetes/pkg/features/kube_features.go". + // Default: nil + // +optional + FeatureGates map[string]bool `json:"featureGates,omitempty"` + + /* the following fields are meant for Node Allocatable */ + + // systemReserved is a set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) + // pairs that describe resources reserved for non-kubernetes components. + // Currently only cpu and memory are supported. + // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. + // Default: nil + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // kubeReserved is a set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs + // that describe resources reserved for kubernetes system components. + // Currently cpu, memory and local storage for root file system are supported. + // See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + // for more details. + // Default: nil + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // The reservedSystemCPUs option specifies the CPU list reserved for the host + // level system threads and kubernetes related threads. This provide a "static" + // CPU list rather than the "dynamic" list by systemReserved and kubeReserved. + // This option does not support systemReservedCgroup or kubeReservedCgroup. + ReservedSystemCPUs string `json:"reservedSystemCPUs,omitempty"` + + /* the following fields are introduced for compatibility with KubeWharf Kubernetes distro */ + + // NumericTopologyAlignResources is a list of resources which need to be aligned numa affinity + // in numeric topology policy. + // Default: [cpu, memory] + // +optional + NumericTopologyAlignResources []string `json:"numericTopologyAlignResources,omitempty"` +} + // GetAndUnmarshalForHttps gets data from the given url and unmarshal it into the given struct. func GetAndUnmarshalForHttps(ctx context.Context, port int, nodeAddress, endpoint, authTokenFile string, v interface{}) error { uri, err := generateURI(port, nodeAddress, endpoint) diff --git a/pkg/util/native/kubelet_test.go b/pkg/util/native/kubelet_test.go index 3417972d2..1d52639eb 100644 --- a/pkg/util/native/kubelet_test.go +++ b/pkg/util/native/kubelet_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" ) func TestGenerateURI(t *testing.T) { @@ -55,7 +54,7 @@ func TestGetAndUnmarshalForHttps(t *testing.T) { authTokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token" type configzWrapper struct { - ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + ComponentConfig KubeletConfiguration `json:"kubeletconfig"` } configz := configzWrapper{}