From 9b67c180da819e5f322ca4392f30ba0faeb998ae Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Tue, 11 Jun 2024 13:23:32 +0200 Subject: [PATCH] Add extra arguments for each service --- src/k8s/api/v1/bootstrap_config.go | 9 ++ src/k8s/api/v1/join_config.go | 15 ++ src/k8s/cmd/k8s/k8s_bootstrap_test.go | 23 +-- .../k8s/testdata/bootstrap-config-full.yaml | 14 ++ src/k8s/pkg/k8sd/app/cluster_util.go | 25 ---- src/k8s/pkg/k8sd/app/hooks_bootstrap.go | 31 ++-- src/k8s/pkg/k8sd/app/hooks_join.go | 21 ++- src/k8s/pkg/k8sd/setup/containerd.go | 8 +- src/k8s/pkg/k8sd/setup/containerd_test.go | 25 +++- src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go | 7 +- .../k8sd/setup/k8s_apiserver_proxy_test.go | 50 ++++++- src/k8s/pkg/k8sd/setup/k8s_dqlite.go | 8 +- src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go | 53 ++++++- src/k8s/pkg/k8sd/setup/kube_apiserver.go | 8 +- src/k8s/pkg/k8sd/setup/kube_apiserver_test.go | 73 +++++++++- .../pkg/k8sd/setup/kube_controller_manager.go | 7 +- .../setup/kube_controller_manager_test.go | 68 ++++++++- src/k8s/pkg/k8sd/setup/kube_proxy.go | 8 +- src/k8s/pkg/k8sd/setup/kube_proxy_test.go | 35 ++++- src/k8s/pkg/k8sd/setup/kube_scheduler.go | 7 +- src/k8s/pkg/k8sd/setup/kube_scheduler_test.go | 51 ++++++- src/k8s/pkg/k8sd/setup/kubelet.go | 16 ++- src/k8s/pkg/k8sd/setup/kubelet_test.go | 134 +++++++++++++++++- src/k8s/pkg/snap/util/services.go | 17 +++ src/k8s/pkg/snap/util/services_test.go | 71 ++++++++++ 25 files changed, 694 insertions(+), 90 deletions(-) diff --git a/src/k8s/api/v1/bootstrap_config.go b/src/k8s/api/v1/bootstrap_config.go index c1c661957..190f7d4e7 100644 --- a/src/k8s/api/v1/bootstrap_config.go +++ b/src/k8s/api/v1/bootstrap_config.go @@ -54,6 +54,15 @@ type BootstrapConfig struct { KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + + // Extra args to add to individual services (set any arg to null to delete) + ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` + ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` + ExtraNodeKubeSchedulerArgs map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"` + ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` + ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` + ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"` + ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"` } func (b *BootstrapConfig) GetDatastoreType() string { return getField(b.DatastoreType) } diff --git a/src/k8s/api/v1/join_config.go b/src/k8s/api/v1/join_config.go index 8324dfdf9..f59b03cb1 100644 --- a/src/k8s/api/v1/join_config.go +++ b/src/k8s/api/v1/join_config.go @@ -25,6 +25,15 @@ type ControlPlaneNodeJoinConfig struct { KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + + // Extra args to add to individual services (set any arg to null to delete) + ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` + ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` + ExtraNodeKubeSchedulerArgs map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"` + ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` + ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` + ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"` + ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"` } type WorkerNodeJoinConfig struct { @@ -34,6 +43,12 @@ type WorkerNodeJoinConfig struct { KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` + + // Extra args to add to individual services (set any arg to null to delete) + ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` + ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` + ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"` + ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"` } func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string { diff --git a/src/k8s/cmd/k8s/k8s_bootstrap_test.go b/src/k8s/cmd/k8s/k8s_bootstrap_test.go index 24783ccbb..de5eda72c 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap_test.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap_test.go @@ -64,14 +64,21 @@ var testCases = []testCase{ }, CloudProvider: utils.Pointer("external"), }, - ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"}, - PodCIDR: utils.Pointer("10.100.0.0/16"), - ServiceCIDR: utils.Pointer("10.200.0.0/16"), - DisableRBAC: utils.Pointer(false), - SecurePort: utils.Pointer(6443), - K8sDqlitePort: utils.Pointer(9090), - DatastoreType: utils.Pointer("k8s-dqlite"), - ExtraSANs: []string{"custom.kubernetes"}, + ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"}, + PodCIDR: utils.Pointer("10.100.0.0/16"), + ServiceCIDR: utils.Pointer("10.200.0.0/16"), + DisableRBAC: utils.Pointer(false), + SecurePort: utils.Pointer(6443), + K8sDqlitePort: utils.Pointer(9090), + DatastoreType: utils.Pointer("k8s-dqlite"), + ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, + ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, + ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, + ExtraNodeKubeProxyArgs: map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")}, + ExtraNodeKubeletArgs: map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")}, + ExtraNodeContainerdArgs: map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")}, + ExtraNodeK8sDqliteArgs: map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")}, }, }, { diff --git a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml index 71d7ed973..07068680c 100644 --- a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml +++ b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml @@ -31,3 +31,17 @@ k8s-dqlite-port: 9090 datastore-type: k8s-dqlite extra-sans: - custom.kubernetes +extra-node-kube-apiserver-args: + --extra-kube-apiserver-arg: extra-kube-apiserver-value +extra-node-kube-controller-manager-args: + --extra-kube-controller-manager-arg: extra-kube-controller-manager-value +extra-node-kube-scheduler-args: + --extra-kube-scheduler-arg: extra-kube-scheduler-value +extra-node-kube-proxy-args: + --extra-kube-proxy-arg: extra-kube-proxy-value +extra-node-kubelet-args: + --extra-kubelet-arg: extra-kubelet-value +extra-node-containerd-args: + --extra-containerd-arg: extra-containerd-value +extra-node-k8s-dqlite-args: + --extra-k8s-dqlite-arg: extra-k8s-dqlite-value diff --git a/src/k8s/pkg/k8sd/app/cluster_util.go b/src/k8s/pkg/k8sd/app/cluster_util.go index 2d5f55b5d..adb7842ad 100644 --- a/src/k8s/pkg/k8sd/app/cluster_util.go +++ b/src/k8s/pkg/k8sd/app/cluster_util.go @@ -3,12 +3,10 @@ package app import ( "context" "fmt" - "net" "path" "github.com/canonical/k8s/pkg/k8sd/pki" "github.com/canonical/k8s/pkg/k8sd/setup" - "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" "github.com/canonical/microcluster/state" @@ -35,29 +33,6 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki } -func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.ClusterConfig, nodeIP net.IP) error { - // Configure services - if err := setup.Containerd(snap, nil); err != nil { - return fmt.Errorf("failed to configure containerd: %w", err) - } - if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints()); err != nil { - return fmt.Errorf("failed to configure kubelet: %w", err) - } - if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR()); err != nil { - return fmt.Errorf("failed to configure kube-proxy: %w", err) - } - if err := setup.KubeControllerManager(snap); err != nil { - return fmt.Errorf("failed to configure kube-controller-manager: %w", err) - } - if err := setup.KubeScheduler(snap); err != nil { - return fmt.Errorf("failed to configure kube-scheduler: %w", err) - } - if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil { - return fmt.Errorf("failed to configure kube-apiserver: %w", err) - } - return nil -} - func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error { // Start services switch datastore { diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index 34033075c..ab90344dd 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -189,17 +189,17 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon } // Worker node services - if err := setup.Containerd(snap, nil); err != nil { + if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil { return fmt.Errorf("failed to configure containerd: %w", err) } - if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil { + if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil { return fmt.Errorf("failed to configure kubelet: %w", err) } - if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR); err != nil { + if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR, joinConfig.ExtraNodeKubeProxyArgs); err != nil { return fmt.Errorf("failed to configure kube-proxy: %w", err) } - if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil { - return fmt.Errorf("failed to configure kube-proxy: %w", err) + if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { + return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) } // TODO(berkayoz): remove the lock on cleanup @@ -344,7 +344,7 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot // Configure datastore switch cfg.Datastore.GetType() { case "k8s-dqlite": - if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil); err != nil { + if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil { return fmt.Errorf("failed to configure k8s-dqlite: %w", err) } case "external": @@ -353,8 +353,23 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot } // Configure services - if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil { - return fmt.Errorf("failed to configure services: %w", err) + if err := setup.Containerd(snap, nil, bootstrapConfig.ExtraNodeContainerdArgs); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) + } + if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) + } + if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) + } + if err := setup.KubeControllerManager(snap, bootstrapConfig.ExtraNodeKubeControllerManagerArgs); err != nil { + return fmt.Errorf("failed to configure kube-controller-manager: %w", err) + } + if err := setup.KubeScheduler(snap, bootstrapConfig.ExtraNodeKubeSchedulerArgs); err != nil { + return fmt.Errorf("failed to configure kube-scheduler: %w", err) + } + if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil { + return fmt.Errorf("failed to configure kube-apiserver: %w", err) } // Write cluster configuration to dqlite diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 2382525f0..38d56a30f 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -134,7 +134,7 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { } address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()) - if err := setup.K8sDqlite(snap, address, cluster); err != nil { + if err := setup.K8sDqlite(snap, address, cluster, joinConfig.ExtraNodeK8sDqliteArgs); err != nil { return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err) } case "external": @@ -143,8 +143,23 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { } // Configure services - if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil { - return fmt.Errorf("failed to configure services: %w", err) + if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) + } + if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) + } + if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), joinConfig.ExtraNodeKubeProxyArgs); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) + } + if err := setup.KubeControllerManager(snap, joinConfig.ExtraNodeKubeControllerManagerArgs); err != nil { + return fmt.Errorf("failed to configure kube-controller-manager: %w", err) + } + if err := setup.KubeScheduler(snap, joinConfig.ExtraNodeKubeSchedulerArgs); err != nil { + return fmt.Errorf("failed to configure kube-scheduler: %w", err) + } + if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil { + return fmt.Errorf("failed to configure kube-apiserver: %w", err) } if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil { diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go index 1e7abab33..e2476c186 100644 --- a/src/k8s/pkg/k8sd/setup/containerd.go +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -113,7 +113,7 @@ func containerdHostConfig(registry types.ContainerdRegistry) containerdHostsConf // Containerd configures configuration and arguments for containerd on the local node. // Optionally, a number of registry mirrors and auths can be configured. -func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error { +func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs map[string]*string) error { configToml, err := os.OpenFile(path.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to open config.toml: %w", err) @@ -138,6 +138,12 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error { return fmt.Errorf("failed to write arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "containerd", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } + cniBinary := path.Join(snap.CNIBinDir(), "cni") if err := utils.CopyFile(snap.CNIPluginsBinary(), cniBinary); err != nil { return fmt.Errorf("failed to copy cni plugin binary: %w", err) diff --git a/src/k8s/pkg/k8sd/setup/containerd_test.go b/src/k8s/pkg/k8sd/setup/containerd_test.go index 50b4c5556..9a898f1c7 100644 --- a/src/k8s/pkg/k8sd/setup/containerd_test.go +++ b/src/k8s/pkg/k8sd/setup/containerd_test.go @@ -12,6 +12,7 @@ import ( "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap/mock" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -40,7 +41,7 @@ func TestContainerd(t *testing.T) { }, } - g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) + g.Expect(setup.EnsureAllDirectories(s)).To(Succeed()) g.Expect(setup.Containerd(s, []types.ContainerdRegistry{ { Host: "docker.io", @@ -53,6 +54,11 @@ func TestContainerd(t *testing.T) { URLs: []string{"https://ghcr.mirror.internal"}, Token: "token", }, + }, map[string]*string{ + "--log-level": utils.Pointer("debug"), + "--metrics": utils.Pointer("true"), + "--address": nil, // This should trigger a delete + "--my-extra-arg": utils.Pointer("my-extra-val"), })).To(Succeed()) t.Run("Config", func(t *testing.T) { @@ -102,10 +108,12 @@ func TestContainerd(t *testing.T) { t.Run("Args", func(t *testing.T) { for key, expectedVal := range map[string]string{ - "--address": path.Join(dir, "containerd-run", "containerd.sock"), - "--config": path.Join(dir, "containerd", "config.toml"), - "--root": path.Join(dir, "containerd-root"), - "--state": path.Join(dir, "containerd-state"), + "--config": path.Join(dir, "containerd", "config.toml"), + "--root": path.Join(dir, "containerd-root"), + "--state": path.Join(dir, "containerd-state"), + "--log-level": "debug", + "--metrics": "true", + "--my-extra-arg": "my-extra-val", } { t.Run(key, func(t *testing.T) { g := NewWithT(t) @@ -114,6 +122,13 @@ func TestContainerd(t *testing.T) { g.Expect(val).To(Equal(expectedVal)) }) } + // --address was deleted by extraArgs + t.Run("--address", func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "containerd", "--address") + g.Expect(err).To(BeNil()) + g.Expect(val).To(BeZero()) + }) }) t.Run("Registries", func(t *testing.T) { diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go index a4c9f4f16..eccb1c540 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go @@ -10,7 +10,7 @@ import ( ) // K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy. -func K8sAPIServerProxy(snap snap.Snap, servers []string) error { +func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*string) error { configFile := path.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json") if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil { return fmt.Errorf("failed to write proxy configuration file: %w", err) @@ -24,5 +24,10 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string) error { return fmt.Errorf("failed to write arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go index 4bca4ddf5..3a9bf54b6 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go @@ -27,7 +27,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) - g.Expect(setup.K8sAPIServerProxy(s, nil)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).To(Succeed()) tests := []struct { key string @@ -45,6 +45,48 @@ func TestK8sApiServerProxy(t *testing.T) { g.Expect(tc.expectedVal).To(Equal(val)) }) } + + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-apiserver-proxy")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) + + extraArgs := map[string]*string{ + "--kubeconfig": utils.Pointer("overridden-kubelet.conf"), + "--listen": nil, // This should trigger a delete + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + g.Expect(setup.K8sAPIServerProxy(s, nil, extraArgs)).To(Succeed()) + + tests := []struct { + key string + expectedVal string + }{ + {key: "--endpoints", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json")}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "overridden-kubelet.conf")}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-apiserver-proxy", tc.key) + g.Expect(err).To(BeNil()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + // --listen was deleted by extraArgs + t.Run("--listen", func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-apiserver-proxy", "--listen") + g.Expect(err).To(BeNil()) + g.Expect(val).To(BeZero()) + }) + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-apiserver-proxy")) g.Expect(err).ToNot(HaveOccurred()) g.Expect(len(args)).To(Equal(len(tests))) @@ -56,7 +98,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceExtraConfigDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed()) }) t.Run("MissingServiceArgumentsDir", func(t *testing.T) { @@ -65,7 +107,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed()) }) t.Run("JSONFileContent", func(t *testing.T) { @@ -76,7 +118,7 @@ func TestK8sApiServerProxy(t *testing.T) { endpoints := []string{"192.168.0.1", "192.168.0.2", "192.168.0.3"} fileName := path.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json") - g.Expect(setup.K8sAPIServerProxy(s, endpoints)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, endpoints, nil)).To(Succeed()) b, err := os.ReadFile(fileName) g.Expect(err).NotTo(HaveOccurred()) diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go index 5e7ca6f3b..6ed7620bd 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go +++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go @@ -15,7 +15,7 @@ type k8sDqliteInit struct { Cluster []string `yaml:"Cluster,omitempty"` } -func K8sDqlite(snap snap.Snap, address string, cluster []string) error { +func K8sDqlite(snap snap.Snap, address string, cluster []string, extraArgs map[string]*string) error { b, err := yaml.Marshal(&k8sDqliteInit{Address: address, Cluster: cluster}) if err != nil { return fmt.Errorf("failed to create init.yaml file for address=%s cluster=%v: %w", address, cluster, err) @@ -31,5 +31,11 @@ func K8sDqlite(snap snap.Snap, address string, cluster []string) error { }, nil); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "k8s-dqlite", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go index 7f2dc733c..077f94e6e 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go +++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go @@ -28,7 +28,7 @@ func TestK8sDqlite(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sDqliteMock) // Call the K8sDqlite setup function with mock arguments - g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"})).To(BeNil()) + g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"}, nil)).To(BeNil()) // Ensure the K8sDqlite arguments file has the expected arguments and values tests := []struct { @@ -53,6 +53,51 @@ func TestK8sDqlite(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setK8sDqliteMock) + + extraArgs := map[string]*string{ + "--my-extra-arg": utils.Pointer("my-extra-val"), + "--listen": nil, + "--storage-dir": utils.Pointer("overridden-storage-dir"), + } + // Call the K8sDqlite setup function with mock arguments + g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"}, extraArgs)).To(BeNil()) + + // Ensure the K8sDqlite arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--storage-dir", expectedVal: "overridden-storage-dir"}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-dqlite", tc.key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(tc.expectedVal)) + }) + } + + // --listen was deleted by extraArgs + t.Run("--listen", func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-dqlite", "--listen") + g.Expect(err).To(BeNil()) + g.Expect(val).To(BeZero()) + }) + + // Ensure the K8sDqlite arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-dqlite")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + t.Run("YAMLFileContents", func(t *testing.T) { g := NewWithT(t) @@ -67,7 +112,7 @@ func TestK8sDqlite(t *testing.T) { "192.168.0.3:1234", } - g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", cluster)).To(BeNil()) + g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", cluster, nil)).To(BeNil()) b, err := os.ReadFile(path.Join(s.Mock.K8sDqliteStateDir, "init.yaml")) g.Expect(err).To(BeNil()) @@ -81,7 +126,7 @@ func TestK8sDqlite(t *testing.T) { s.Mock.K8sDqliteStateDir = "nonexistent" - g.Expect(setup.K8sDqlite(s, "", []string{})).ToNot(Succeed()) + g.Expect(setup.K8sDqlite(s, "", []string{}, nil)).ToNot(Succeed()) }) t.Run("MissingArgsDir", func(t *testing.T) { @@ -91,6 +136,6 @@ func TestK8sDqlite(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.K8sDqlite(s, "", []string{})).ToNot(Succeed()) + g.Expect(setup.K8sDqlite(s, "", []string{}, nil)).ToNot(Succeed()) }) } diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go index 09d797f27..b932311f5 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -47,7 +47,7 @@ var ( ) // KubeAPIServer configures kube-apiserver on the local node. -func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string) error { +func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string, extraArgs map[string]*string) error { authTokenWebhookConfigFile := path.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf") authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { @@ -105,5 +105,11 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, deleteArgs); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go index 6eede1f05..20e06adcb 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go @@ -36,7 +36,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -91,7 +91,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -132,13 +132,76 @@ func TestKubeAPIServer(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) + + extraArgs := map[string]*string{ + "--allow-privileged": nil, + "--secure-port": utils.Pointer("1337"), + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the KubeAPIServer setup function with mock arguments + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", extraArgs)).To(BeNil()) + + // Ensure the kube-apiserver arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--authentication-token-webhook-config-file", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "auth-token-webhook.conf")}, + {key: "--authorization-mode", expectedVal: "Node,RBAC"}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, + {key: "--enable-admission-plugins", expectedVal: "NodeRestriction"}, + {key: "--kubelet-certificate-authority", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--kubelet-client-certificate", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.crt")}, + {key: "--kubelet-client-key", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.key")}, + {key: "--kubelet-preferred-address-types", expectedVal: "InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP"}, + {key: "--secure-port", expectedVal: "1337"}, + {key: "--service-account-issuer", expectedVal: "https://kubernetes.default.svc"}, + {key: "--service-account-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")}, + {key: "--service-account-signing-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")}, + {key: "--service-cluster-ip-range", expectedVal: "10.0.0.0/24"}, + {key: "--tls-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver.crt")}, + {key: "--tls-cipher-suites", expectedVal: apiserverTLSCipherSuites}, + {key: "--tls-private-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver.key")}, + {key: "--etcd-servers", expectedVal: fmt.Sprintf("unix://%s", path.Join(s.Mock.K8sDqliteStateDir, "k8s-dqlite.sock"))}, + {key: "--requestheader-client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-ca.crt")}, + {key: "--requestheader-allowed-names", expectedVal: "front-proxy-client"}, + {key: "--requestheader-extra-headers-prefix", expectedVal: "X-Remote-Extra-"}, + {key: "--requestheader-group-headers", expectedVal: "X-Remote-Group"}, + {key: "--requestheader-username-headers", expectedVal: "X-Remote-User"}, + {key: "--proxy-client-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-client.crt")}, + {key: "--proxy-client-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-client.key")}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-apiserver", tc.key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(tc.expectedVal)) + }) + } + // Ensure that the allow-privileged argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-apiserver", "--allow-privileged") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kube-apiserver arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) t.Run("ArgsDualstack", func(t *testing.T) { g := NewWithT(t) s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--service-cluster-ip-range")).To(Equal("10.0.0.0/24,fd01::/64")) _, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -151,7 +214,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--etcd-servers")).To(Equal("datastoreurl1,datastoreurl2")) _, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -165,7 +228,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Attempt to configure kube-apiserver with an unsupported datastore - err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC") + err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC", nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(ContainSubstring("unsupported datastore"))) }) diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go index a76bb6643..5b3bba22c 100644 --- a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go +++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go @@ -10,7 +10,7 @@ import ( ) // KubeControllerManager configures kube-controller-manager on the local node. -func KubeControllerManager(snap snap.Snap) error { +func KubeControllerManager(snap snap.Snap, extraArgs map[string]*string) error { args := map[string]string{ "--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), "--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), @@ -30,5 +30,10 @@ func KubeControllerManager(snap snap.Snap) error { if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", args, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go index b5c44f800..e6878b2a1 100644 --- a/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go @@ -31,7 +31,7 @@ func TestKubeControllerManager(t *testing.T) { os.Create(path.Join(s.Mock.KubernetesPKIDir, "ca.key")) // Call the kube controller manager setup function - g.Expect(setup.KubeControllerManager(s)).To(BeNil()) + g.Expect(setup.KubeControllerManager(s, nil)).To(BeNil()) // Ensure the kube controller manager arguments file has the expected arguments and values tests := []struct { @@ -67,7 +67,7 @@ func TestKubeControllerManager(t *testing.T) { t.Run("MissingArgsDir", func(t *testing.T) { g := NewWithT(t) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeControllerManager(s)).ToNot(Succeed()) + g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed()) }) }) @@ -78,7 +78,7 @@ func TestKubeControllerManager(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeControllerManagerMock) // Call the kube controller manager setup function - g.Expect(setup.KubeControllerManager(s)).To(BeNil()) + g.Expect(setup.KubeControllerManager(s, nil)).To(BeNil()) // Ensure the kube controller manager arguments file has the expected arguments and values tests := []struct { @@ -112,7 +112,67 @@ func TestKubeControllerManager(t *testing.T) { t.Run("MissingArgsDir", func(t *testing.T) { g := NewWithT(t) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeControllerManager(s)).ToNot(Succeed()) + g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed()) + }) + }) + + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeControllerManagerMock) + + // Create ca.key so that cluster-signing-cert-file and cluster-signing-key-file are added to the arguments + os.Create(path.Join(s.Mock.KubernetesPKIDir, "ca.key")) + + extraArgs := map[string]*string{ + "--leader-elect-lease-duration": nil, + "--profiling": utils.Pointer("true"), + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the kube controller manager setup function + g.Expect(setup.KubeControllerManager(s, extraArgs)).To(BeNil()) + + // Ensure the kube controller manager arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--authentication-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")}, + {key: "--authorization-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")}, + {key: "--leader-elect-renew-deadline", expectedVal: "15s"}, + {key: "--profiling", expectedVal: "true"}, + {key: "--root-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--service-account-private-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")}, + {key: "--use-service-account-credentials", expectedVal: "true"}, + {key: "--cluster-signing-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--cluster-signing-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.key")}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-controller-manager", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the leader-elect-lease-duration argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-controller-manager", "--leader-elect-lease-duration") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kube controller manager arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-controller-manager")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + + t.Run("MissingArgsDir", func(t *testing.T) { + g := NewWithT(t) + s.Mock.ServiceArgumentsDir = "nonexistent" + g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed()) }) }) } diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy.go b/src/k8s/pkg/k8sd/setup/kube_proxy.go index a1f7a59f6..6731b2925 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy.go @@ -11,7 +11,7 @@ import ( ) // KubeProxy configures kube-proxy on the local node. -func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string) error { +func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string, extraArgs map[string]*string) error { serviceArgs := map[string]string{ "--cluster-cidr": podCIDR, "--healthz-bind-address": "127.0.0.1", @@ -33,5 +33,11 @@ func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR str if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", serviceArgs, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go index 17d8ac382..69c71d921 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go @@ -9,6 +9,7 @@ import ( "github.com/canonical/k8s/pkg/k8sd/setup" "github.com/canonical/k8s/pkg/snap/mock" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -30,11 +31,10 @@ func TestKubeProxy(t *testing.T) { g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) t.Run("Args", func(t *testing.T) { - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil()) for key, expectedVal := range map[string]string{ "--cluster-cidr": "10.1.0.0/16", - "--healthz-bind-address": "127.0.0.1", "--hostname-override": "myhostname", "--kubeconfig": path.Join(dir, "kubernetes", "proxy.conf"), "--profiling": "false", @@ -49,9 +49,38 @@ func TestKubeProxy(t *testing.T) { } }) + t.Run("WithExtraArgs", func(t *testing.T) { + extraArgs := map[string]*string{ + "--hostname-override": utils.Pointer("myoverriddenhostname"), + "--healthz-bind-address": nil, + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", extraArgs)).To(BeNil()) + + for key, expectedVal := range map[string]string{ + "--cluster-cidr": "10.1.0.0/16", + "--hostname-override": "myoverriddenhostname", + "--kubeconfig": path.Join(dir, "kubernetes", "proxy.conf"), + "--profiling": "false", + "--conntrack-max-per-core": "", + "--my-extra-arg": "my-extra-val", + } { + t.Run(key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-proxy", key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(expectedVal)) + }) + } + // Ensure that the healthz-bind-address argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-proxy", "--healthz-bind-address") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + }) + s.Mock.OnLXD = true t.Run("ArgsOnLXD", func(t *testing.T) { - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil()) for key, expectedVal := range map[string]string{ "--conntrack-max-per-core": "0", diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler.go b/src/k8s/pkg/k8sd/setup/kube_scheduler.go index 1cc6ae5b4..da151a0bc 100644 --- a/src/k8s/pkg/k8sd/setup/kube_scheduler.go +++ b/src/k8s/pkg/k8sd/setup/kube_scheduler.go @@ -9,7 +9,7 @@ import ( ) // KubeScheduler configures kube-scheduler on the local node. -func KubeScheduler(snap snap.Snap) error { +func KubeScheduler(snap snap.Snap, extraArgs map[string]*string) error { if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", map[string]string{ "--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), "--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), @@ -20,5 +20,10 @@ func KubeScheduler(snap snap.Snap) error { }, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go b/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go index f4e9f62b1..d478bd373 100644 --- a/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go @@ -26,7 +26,7 @@ func TestKubeScheduler(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeSchedulerMock) // Call the kube scheduler setup function - g.Expect(setup.KubeScheduler(s)).To(BeNil()) + g.Expect(setup.KubeScheduler(s, nil)).To(BeNil()) // Ensure the kube scheduler arguments file has the expected arguments and values tests := []struct { @@ -56,6 +56,53 @@ func TestKubeScheduler(t *testing.T) { }) + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeSchedulerMock) + + extraArgs := map[string]*string{ + "--leader-elect-lease-duration": nil, + "--profiling": utils.Pointer("true"), + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the kube scheduler setup function + g.Expect(setup.KubeScheduler(s, extraArgs)).To(BeNil()) + + // Ensure the kube scheduler arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--authentication-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")}, + {key: "--authorization-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")}, + {key: "--leader-elect-renew-deadline", expectedVal: "15s"}, + {key: "--profiling", expectedVal: "true"}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-scheduler", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the leader-elect-lease-duration argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-scheduler", "--leader-elect-lease-duration") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kube scheduler arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-scheduler")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + + }) + t.Run("MissingArgsDir", func(t *testing.T) { g := NewWithT(t) @@ -63,6 +110,6 @@ func TestKubeScheduler(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeScheduler(s)).ToNot(Succeed()) + g.Expect(setup.KubeScheduler(s, nil)).ToNot(Succeed()) }) } diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go index 7ad85fcdf..bac5da7d2 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet.go +++ b/src/k8s/pkg/k8sd/setup/kubelet.go @@ -30,17 +30,17 @@ var kubeletWorkerLabels = []string{ } // KubeletControlPlane configures kubelet on a control plane node. -func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, registerWithTaints []string) error { - return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, registerWithTaints, append(kubeletControlPlaneLabels, kubeletWorkerLabels...)) +func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, registerWithTaints []string, extraArgs map[string]*string) error { + return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, registerWithTaints, append(kubeletControlPlaneLabels, kubeletWorkerLabels...), extraArgs) } // KubeletWorker configures kubelet on a worker node. -func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string) error { - return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels) +func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, extraArgs map[string]*string) error { + return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels, extraArgs) } // kubelet configures kubelet on the local node. -func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, taints []string, labels []string) error { +func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, taints []string, labels []string, extraArgs map[string]*string) error { args := map[string]string{ "--anonymous-auth": "false", "--authentication-token-webhook": "true", @@ -74,5 +74,11 @@ func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", args, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kubelet_test.go b/src/k8s/pkg/k8sd/setup/kubelet_test.go index 59d11e980..76686491a 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet_test.go +++ b/src/k8s/pkg/k8sd/setup/kubelet_test.go @@ -46,7 +46,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet control plane setup function - g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).To(Succeed()) + g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, nil)).To(Succeed()) // Ensure the kubelet arguments file has the expected arguments and values tests := []struct { @@ -69,8 +69,8 @@ func TestKubelet(t *testing.T) { {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir}, {key: "--serialize-image-pulls", expectedVal: "false"}, {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites}, - {key: "--cloud-provider", expectedVal: "provider"}, {key: "--cluster-dns", expectedVal: "10.152.1.1"}, + {key: "--cloud-provider", expectedVal: "provider"}, {key: "--cluster-domain", expectedVal: "test-cluster.local"}, {key: "--node-ip", expectedVal: "192.168.0.1"}, } @@ -89,6 +89,67 @@ func TestKubelet(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("ControlPlaneWithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeletMock) + + extraArgs := map[string]*string{ + "--cluster-domain": utils.Pointer("override.local"), + "--cloud-provider": nil, // This should trigger a delete + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the kubelet control plane setup function + g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, extraArgs)).To(Succeed()) + + // Ensure the kubelet arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--anonymous-auth", expectedVal: "false"}, + {key: "--authentication-token-webhook", expectedVal: "true"}, + {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, + {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, + {key: "--fail-swap-on", expectedVal: "false"}, + {key: "--hostname-override", expectedVal: "dev"}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")}, + {key: "--node-labels", expectedVal: expectedControlPlaneLabels}, + {key: "--read-only-port", expectedVal: "0"}, + {key: "--register-with-taints", expectedVal: ""}, + {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir}, + {key: "--serialize-image-pulls", expectedVal: "false"}, + {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites}, + {key: "--cluster-dns", expectedVal: "10.152.1.1"}, + // Overwritten by extraArgs + {key: "--cluster-domain", expectedVal: "override.local"}, + {key: "--node-ip", expectedVal: "192.168.0.1"}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kubelet", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the cloud-provider argument was deleted + val, err := snaputil.GetServiceArgument(s, "kubelet", "--cloud-provider") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kubelet arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kubelet")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + t.Run("ControlPlaneArgsNoOptional", func(t *testing.T) { g := NewWithT(t) @@ -96,7 +157,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet control plane setup function - g.Expect(setup.KubeletControlPlane(s, "dev", nil, "", "", "", nil)).To(BeNil()) + g.Expect(setup.KubeletControlPlane(s, "dev", nil, "", "", "", nil, nil)).To(BeNil()) tests := []struct { key string @@ -141,7 +202,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet worker setup function - g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider")).To(BeNil()) + g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).To(BeNil()) // Ensure the kubelet arguments file has the expected arguments and values tests := []struct { @@ -184,6 +245,65 @@ func TestKubelet(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("WorkerWithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeletMock) + + extraArgs := map[string]*string{ + "--cluster-domain": utils.Pointer("override.local"), + "--cloud-provider": nil, + } + + // Call the kubelet worker setup function + g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", extraArgs)).To(BeNil()) + + // Ensure the kubelet arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--anonymous-auth", expectedVal: "false"}, + {key: "--authentication-token-webhook", expectedVal: "true"}, + {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, + {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, + {key: "--fail-swap-on", expectedVal: "false"}, + {key: "--hostname-override", expectedVal: "dev"}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")}, + {key: "--node-labels", expectedVal: expectedWorkerLabels}, + {key: "--read-only-port", expectedVal: "0"}, + {key: "--register-with-taints", expectedVal: ""}, + {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir}, + {key: "--serialize-image-pulls", expectedVal: "false"}, + {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites}, + {key: "--cluster-dns", expectedVal: "10.152.1.1"}, + {key: "--cluster-domain", expectedVal: "override.local"}, + {key: "--node-ip", expectedVal: "192.168.0.1"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kubelet", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the cloud-provider argument was deleted + val, err := snaputil.GetServiceArgument(s, "kubelet", "--cloud-provider") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kubelet arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kubelet")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + t.Run("WorkerArgsNoOptional", func(t *testing.T) { g := NewWithT(t) @@ -191,7 +311,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet worker setup function - g.Expect(setup.KubeletWorker(s, "dev", nil, "", "", "")).To(BeNil()) + g.Expect(setup.KubeletWorker(s, "dev", nil, "", "", "", nil)).To(BeNil()) // Ensure the kubelet arguments file has the expected arguments and values tests := []struct { @@ -236,7 +356,7 @@ func TestKubelet(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).ToNot(Succeed()) + g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, nil)).ToNot(Succeed()) }) t.Run("WorkerNoArgsDir", func(t *testing.T) { @@ -245,6 +365,6 @@ func TestKubelet(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider")).ToNot(Succeed()) + g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).ToNot(Succeed()) }) } diff --git a/src/k8s/pkg/snap/util/services.go b/src/k8s/pkg/snap/util/services.go index 56be97bf2..d7587207e 100644 --- a/src/k8s/pkg/snap/util/services.go +++ b/src/k8s/pkg/snap/util/services.go @@ -75,3 +75,20 @@ func StopK8sDqliteServices(ctx context.Context, snap snap.Snap) error { } return nil } + +// ServiceArgsFromMap processes a map of string pointers and categorizes them into update and delete lists. +// - If the value pointer is nil, it adds the argument name to the delete list. +// - If the value pointer is not nil, it adds the argument and its value to the update map. +func ServiceArgsFromMap(args map[string]*string) (map[string]string, []string) { + updateArgs := make(map[string]string) + deleteArgs := make([]string, 0) + + for arg, val := range args { + if val == nil { + deleteArgs = append(deleteArgs, arg) + } else { + updateArgs[arg] = *val + } + } + return updateArgs, deleteArgs +} diff --git a/src/k8s/pkg/snap/util/services_test.go b/src/k8s/pkg/snap/util/services_test.go index 203012d81..70ae0bc75 100644 --- a/src/k8s/pkg/snap/util/services_test.go +++ b/src/k8s/pkg/snap/util/services_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/canonical/k8s/pkg/snap/mock" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -108,3 +109,73 @@ func TestStopK8sDqliteServices(t *testing.T) { g.Expect(StopK8sDqliteServices(context.Background(), mock)).NotTo(Succeed()) }) } + +func TestServiceArgsFromMap(t *testing.T) { + tests := []struct { + name string + input map[string]*string + expected struct { + updateArgs map[string]string + deleteArgs []string + } + }{ + { + name: "NilValue", + input: map[string]*string{"arg1": nil}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{}, + deleteArgs: []string{"arg1"}, + }, + }, + { + name: "EmptyString", // Should be threated as normal string + input: map[string]*string{"arg1": utils.Pointer("")}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": ""}, + deleteArgs: []string{}, + }, + }, + { + name: "NonEmptyString", + input: map[string]*string{"arg1": utils.Pointer("value1")}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": "value1"}, + deleteArgs: []string{}, + }, + }, + { + name: "MixedValues", + input: map[string]*string{ + "arg1": utils.Pointer("value1"), + "arg2": utils.Pointer(""), + "arg3": nil, + }, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": "value1", "arg2": ""}, + deleteArgs: []string{"arg3"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + updateArgs, deleteArgs := ServiceArgsFromMap(tt.input) + g.Expect(updateArgs).To(Equal(tt.expected.updateArgs)) + g.Expect(deleteArgs).To(Equal(tt.expected.deleteArgs)) + }) + } +}