all settings

abhay-krishna · Jun 4, 2024 · ba9c66f · ba9c66f
1 parent 94ead19
commit ba9c66f
Show file tree

Hide file tree

Showing 9 changed files with 205 additions and 33 deletions.
diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go
@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	bootstraputil "k8s.io/cluster-bootstrap/token/util"
 )
@@ -232,7 +233,7 @@ type BottlerocketKubernetesSettings struct {
 	ContainerLogMaxSize string `json:"containerLogMaxSize,omitempty"`
 
 	// CpuCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits
-	CpuCFSQuota bool `json:"cpuCFSQuota,omitempty"`
+	CpuCFSQuota *bool `json:"cpuCFSQuota,omitempty"`
 
 	// CpuManagerPolicy is the name of the policy to use.
 	CpuManagerPolicy string `json:"cpuManagerPolicy,omitempty"`
@@ -242,7 +243,7 @@ type BottlerocketKubernetesSettings struct {
 	CpuManagerPolicyOptions map[string]string `json:"cpuManagerPolicyOptions,omitempty"`
 
 	// CpuManagerReconcilePeriod is the reconciliation period for the CPU Manager.
-	// CpuManagerReconcilePeriod *v1.Duration `json:"cpuManagerReconcilePeriod,omitempty"`
+	CpuManagerReconcilePeriod *v1.Duration `json:"cpuManagerReconcilePeriod,omitempty"`
 
 	// EventBurst is the maximum size of a burst of event creations.
 	EventBurst int `json:"eventBurst,omitempty"`
@@ -300,15 +301,15 @@ type BottlerocketKubernetesSettings struct {
 	RegistryPullQPS int `json:"registryPullQPS,omitempty"`
 
 	// ServerTLSBootstrap enables server certificate bootstrap.
-	ServerTLSBootstrap bool `json:"serverTLSBootstrap,omitempty"`
+	ServerTLSBootstrap *bool `json:"serverTLSBootstrap,omitempty"`
 
 	// ShutdownGracePeriod specifies the total duration that the node should delay
 	// the shutdown and total grace period for pod termination during a node shutdown.
-	// ShutdownGracePeriod *v1.Duration `json:"shutdownGracePeriod,omitempty"`
+	ShutdownGracePeriod *v1.Duration `json:"shutdownGracePeriod,omitempty"`
 
 	// ShutdownGracePeriodCriticalPods specifies the duration used to terminate
 	// critical pods during a node shutdown.
-	// ShutdownGracePeriodCriticalPods *v1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"`
+	ShutdownGracePeriodCriticalPods *v1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"`
 
 	// SystemReserved is a set of ResourceName=ResourceQuantity pairs that describe
 	// resources reserved for non-kubernetes components.

diff --git a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go
diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml
diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml
diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go
@@ -23,6 +23,9 @@ allowed-unsafe-sysctls = [{{stringsJoin .AllowedUnsafeSysctls ", " }}]
 {{- if .ClusterDNSIPs }}
 cluster-dns-ip = [{{stringsJoin .ClusterDNSIPs ", " }}]
 {{- end -}}
+{{- if ne .CpuCFSQuota nil }}
+cpu-cfs-quota-enforced = {{ .CpuCFSQuota }}
+{{- end -}}
 {{- if .MaxPods }}
 max-pods = {{.MaxPods}}
 {{- end -}}
@@ -42,6 +45,9 @@ cpu-manager-policy-options = ["{{ $key }}"]
 {{- end }}
 {{- end }}
 {{- end }}
+{{- if .CpuManagerReconcilePeriod }}
+cpu-manager-reconcile-period = {{.CpuManagerReconcilePeriod}}
+{{- end -}}
 {{- if .EventBurst }}
 event-burst = {{.EventBurst}}
 {{- end -}}
@@ -75,6 +81,12 @@ registry-burst = {{.RegistryBurst}}
 {{- if .RegistryPullQPS }}
 registry-qps = {{.RegistryPullQPS}}
 {{- end -}}
+{{- if .ShutdownGracePeriod }}
+shutdown-grace-period = {{.ShutdownGracePeriod}}
+{{- end -}}
+{{- if .ShutdownGracePeriodCriticalPods }}
+shutdown-grace-period-for-critical-pods = {{.ShutdownGracePeriodCriticalPods}}
+{{- end -}}
 {{- if .TopologyManagerPolicy }}
 topology-manager-policy = "{{.TopologyManagerPolicy}}"
 {{- end -}}

diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go
@@ -2,9 +2,12 @@ package bottlerocket
 
 import (
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 )
@@ -398,10 +401,12 @@ authentication-mode = "tls"
 server-tls-bootstrap = false
 pod-infra-container-image = "PAUSE_REPO:PAUSE_TAG"
 provider-id = "PROVIDERID"
+cpu-cfs-quota-enforced = false
 container-log-max-files = 10
 container-log-max-size = "5Mi"
 cpu-manager-policy = "static"
 cpu-manager-policy-options = ["full-pcpus-only"]
+cpu-manager-reconcile-period = 10s
 event-burst = 200
 event-qps = 100
 eviction-max-pod-grace-period = 10
@@ -412,6 +417,8 @@ memory-manager-policy = "Static"
 pod-pids-limit = 10
 registry-burst = 11
 registry-qps = 1
+shutdown-grace-period = 15s
+shutdown-grace-period-for-critical-pods = 20s
 topology-manager-policy = "restricted"
 topology-manager-scope = "pod"
 [settings.kubernetes.eviction-hard]
@@ -752,12 +759,15 @@ func TestGetBottlerocketNodeUserData(t *testing.T) {
 						ClusterDomain:        "cluster.local2",
 						ContainerLogMaxFiles: 10,
 						ContainerLogMaxSize:  "5Mi",
-						// CpuCFSQuota:          true,
+						CpuCFSQuota:          pointer.Bool(false),
 						CpuManagerPolicy:     "static",
 						CpuManagerPolicyOptions: map[string]string{
 							"full-pcpus-only": "true",
 						},
-						EventBurst: 200,
+						CpuManagerReconcilePeriod: &v1.Duration{
+							Duration: 10 * time.Second,
+						},
+						EventBurst:     200,
 						EventRecordQPS: 100,
 						EvictionHard: map[string]string{
 							"memory.available": "15%",
@@ -770,26 +780,28 @@ func TestGetBottlerocketNodeUserData(t *testing.T) {
 							"memory.available": "30s",
 						},
 						ImageGCHighThresholdPercent: 26,
-						ImageGCLowThresholdPercent: 19,
-						KubeAPIBurst: 80,
+						ImageGCLowThresholdPercent:  19,
+						KubeAPIBurst:                80,
 						KubeReserved: map[string]string{
 							"cpu": "20m",
 						},
 						MemoryManagerPolicy: "Static",
-						PodPidsLimit: 10,
-						RegistryBurst: 11,
-						RegistryPullQPS: 1,
-						// ShutdownGracePeriod: &v1.Duration{
-						// 	Duration: 10 * time.Second,
-						// },
+						PodPidsLimit:        10,
+						RegistryBurst:       11,
+						RegistryPullQPS:     1,
+						ShutdownGracePeriod: &v1.Duration{
+							Duration: 15 * time.Second,
+						},
+						ShutdownGracePeriodCriticalPods: &v1.Duration{
+							Duration: 20 * time.Second,
+						},
 						SystemReserved: map[string]string{
 							"cpu":               "10m",
 							"ephemeral-storage": "1Gi",
 							"memory":            "100Mi",
 						},
-						// ServerTLSBootstrap: true,
 						TopologyManagerPolicy: "restricted",
-						TopologyManagerScope: "pod",
+						TopologyManagerScope:  "pod",
 					},
 				},
 			},

diff --git a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go
@@ -68,10 +68,10 @@ type BottlerocketSettingsInput struct {
 	ClusterDomain                   string
 	ContainerLogMaxFiles            int
 	ContainerLogMaxSize             string
-	CpuCFSQuota                     bool
+	CpuCFSQuota                     *bool
 	CpuManagerPolicy                string
 	CpuManagerPolicyOptions         map[string]string
-	// CpuManagerReconcilePeriod       string
+	CpuManagerReconcilePeriod       string
 	EventBurst                      int
 	EventRecordQPS                  int
 	EvictionHard                    map[string]string
@@ -87,9 +87,9 @@ type BottlerocketSettingsInput struct {
 	PodPidsLimit                    int64
 	RegistryBurst                   int
 	RegistryPullQPS                 int
-	ServerTLSBootstrap              bool
-	// ShutdownGracePeriod             string
-	// ShutdownGracePeriodCriticalPods string
+	ServerTLSBootstrap              *bool
+	ShutdownGracePeriod             string
+	ShutdownGracePeriodCriticalPods string
 	SystemReserved                  map[string]string
 	TopologyManagerPolicy           string
 	TopologyManagerScope            string
@@ -335,12 +335,14 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot
 			bottlerocketInput.ClusterDomain = config.BottlerocketSettings.Kubernetes.ClusterDomain
 			bottlerocketInput.ContainerLogMaxFiles = config.BottlerocketSettings.Kubernetes.ContainerLogMaxFiles
 			bottlerocketInput.ContainerLogMaxSize = config.BottlerocketSettings.Kubernetes.ContainerLogMaxSize
-			// bottlerocketInput.CpuCFSQuota = config.BottlerocketSettings.Kubernetes.CpuCFSQuota
+			if config.BottlerocketSettings.Kubernetes.CpuCFSQuota != nil {
+				bottlerocketInput.CpuCFSQuota = config.BottlerocketSettings.Kubernetes.CpuCFSQuota
+			}
 			bottlerocketInput.CpuManagerPolicy = config.BottlerocketSettings.Kubernetes.CpuManagerPolicy
 			bottlerocketInput.CpuManagerPolicyOptions = config.BottlerocketSettings.Kubernetes.CpuManagerPolicyOptions
-			// if config.BottlerocketSettings.Kubernetes.CpuManagerReconcilePeriod != nil {
-			// bottlerocketInput.CpuManagerReconcilePeriod = config.BottlerocketSettings.Kubernetes.CpuManagerReconcilePeriod.String()
-			// }
+			if config.BottlerocketSettings.Kubernetes.CpuManagerReconcilePeriod != nil {
+				bottlerocketInput.CpuManagerReconcilePeriod = config.BottlerocketSettings.Kubernetes.CpuManagerReconcilePeriod.Duration.String()
+			}
 			bottlerocketInput.EventBurst = config.BottlerocketSettings.Kubernetes.EventBurst
 			bottlerocketInput.EventRecordQPS = config.BottlerocketSettings.Kubernetes.EventRecordQPS
 			bottlerocketInput.EvictionHard = config.BottlerocketSettings.Kubernetes.EvictionHard
@@ -357,13 +359,15 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot
 			bottlerocketInput.PodPidsLimit = config.BottlerocketSettings.Kubernetes.PodPidsLimit
 			bottlerocketInput.RegistryBurst = config.BottlerocketSettings.Kubernetes.RegistryBurst
 			bottlerocketInput.RegistryPullQPS = config.BottlerocketSettings.Kubernetes.RegistryPullQPS
-			// bottlerocketInput.ServerTLSBootstrap = config.BottlerocketSettings.Kubernetes.ServerTLSBootstrap
-			// if config.BottlerocketSettings.Kubernetes.ShutdownGracePeriod != nil {
-			// bottlerocketInput.ShutdownGracePeriod = config.BottlerocketSettings.Kubernetes.ShutdownGracePeriod.String()
-			// }
-			// if config.BottlerocketSettings.Kubernetes.ShutdownGracePeriodCriticalPods != nil {
-			// bottlerocketInput.ShutdownGracePeriodCriticalPods = config.BottlerocketSettings.Kubernetes.ShutdownGracePeriodCriticalPods.String()
-			// }
+			if config.BottlerocketSettings.Kubernetes.ServerTLSBootstrap != nil {
+				bottlerocketInput.ServerTLSBootstrap = config.BottlerocketSettings.Kubernetes.ServerTLSBootstrap
+			}
+			if config.BottlerocketSettings.Kubernetes.ShutdownGracePeriod != nil {
+				bottlerocketInput.ShutdownGracePeriod = config.BottlerocketSettings.Kubernetes.ShutdownGracePeriod.Duration.String()
+			}
+			if config.BottlerocketSettings.Kubernetes.ShutdownGracePeriodCriticalPods != nil {
+				bottlerocketInput.ShutdownGracePeriodCriticalPods = config.BottlerocketSettings.Kubernetes.ShutdownGracePeriodCriticalPods.Duration.String()
+			}
 			bottlerocketInput.SystemReserved = config.BottlerocketSettings.Kubernetes.SystemReserved
 			bottlerocketInput.TopologyManagerPolicy = config.BottlerocketSettings.Kubernetes.TopologyManagerPolicy
 			bottlerocketInput.TopologyManagerScope = config.BottlerocketSettings.Kubernetes.TopologyManagerScope