From 9b67c180da819e5f322ca4392f30ba0faeb998ae Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Tue, 11 Jun 2024 13:23:32 +0200 Subject: [PATCH 1/6] Add extra arguments for each service --- src/k8s/api/v1/bootstrap_config.go | 9 ++ src/k8s/api/v1/join_config.go | 15 ++ src/k8s/cmd/k8s/k8s_bootstrap_test.go | 23 +-- .../k8s/testdata/bootstrap-config-full.yaml | 14 ++ src/k8s/pkg/k8sd/app/cluster_util.go | 25 ---- src/k8s/pkg/k8sd/app/hooks_bootstrap.go | 31 ++-- src/k8s/pkg/k8sd/app/hooks_join.go | 21 ++- src/k8s/pkg/k8sd/setup/containerd.go | 8 +- src/k8s/pkg/k8sd/setup/containerd_test.go | 25 +++- src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go | 7 +- .../k8sd/setup/k8s_apiserver_proxy_test.go | 50 ++++++- src/k8s/pkg/k8sd/setup/k8s_dqlite.go | 8 +- src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go | 53 ++++++- src/k8s/pkg/k8sd/setup/kube_apiserver.go | 8 +- src/k8s/pkg/k8sd/setup/kube_apiserver_test.go | 73 +++++++++- .../pkg/k8sd/setup/kube_controller_manager.go | 7 +- .../setup/kube_controller_manager_test.go | 68 ++++++++- src/k8s/pkg/k8sd/setup/kube_proxy.go | 8 +- src/k8s/pkg/k8sd/setup/kube_proxy_test.go | 35 ++++- src/k8s/pkg/k8sd/setup/kube_scheduler.go | 7 +- src/k8s/pkg/k8sd/setup/kube_scheduler_test.go | 51 ++++++- src/k8s/pkg/k8sd/setup/kubelet.go | 16 ++- src/k8s/pkg/k8sd/setup/kubelet_test.go | 134 +++++++++++++++++- src/k8s/pkg/snap/util/services.go | 17 +++ src/k8s/pkg/snap/util/services_test.go | 71 ++++++++++ 25 files changed, 694 insertions(+), 90 deletions(-) diff --git a/src/k8s/api/v1/bootstrap_config.go b/src/k8s/api/v1/bootstrap_config.go index c1c661957..190f7d4e7 100644 --- a/src/k8s/api/v1/bootstrap_config.go +++ b/src/k8s/api/v1/bootstrap_config.go @@ -54,6 +54,15 @@ type BootstrapConfig struct { KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + + // Extra args to add to individual services (set any arg to null to delete) + ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` + ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` + ExtraNodeKubeSchedulerArgs map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"` + ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` + ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` + ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"` + ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"` } func (b *BootstrapConfig) GetDatastoreType() string { return getField(b.DatastoreType) } diff --git a/src/k8s/api/v1/join_config.go b/src/k8s/api/v1/join_config.go index 8324dfdf9..f59b03cb1 100644 --- a/src/k8s/api/v1/join_config.go +++ b/src/k8s/api/v1/join_config.go @@ -25,6 +25,15 @@ type ControlPlaneNodeJoinConfig struct { KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + + // Extra args to add to individual services (set any arg to null to delete) + ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` + ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` + ExtraNodeKubeSchedulerArgs map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"` + ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` + ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` + ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"` + ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"` } type WorkerNodeJoinConfig struct { @@ -34,6 +43,12 @@ type WorkerNodeJoinConfig struct { KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` + + // Extra args to add to individual services (set any arg to null to delete) + ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` + ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` + ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"` + ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"` } func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string { diff --git a/src/k8s/cmd/k8s/k8s_bootstrap_test.go b/src/k8s/cmd/k8s/k8s_bootstrap_test.go index 24783ccbb..de5eda72c 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap_test.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap_test.go @@ -64,14 +64,21 @@ var testCases = []testCase{ }, CloudProvider: utils.Pointer("external"), }, - ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"}, - PodCIDR: utils.Pointer("10.100.0.0/16"), - ServiceCIDR: utils.Pointer("10.200.0.0/16"), - DisableRBAC: utils.Pointer(false), - SecurePort: utils.Pointer(6443), - K8sDqlitePort: utils.Pointer(9090), - DatastoreType: utils.Pointer("k8s-dqlite"), - ExtraSANs: []string{"custom.kubernetes"}, + ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"}, + PodCIDR: utils.Pointer("10.100.0.0/16"), + ServiceCIDR: utils.Pointer("10.200.0.0/16"), + DisableRBAC: utils.Pointer(false), + SecurePort: utils.Pointer(6443), + K8sDqlitePort: utils.Pointer(9090), + DatastoreType: utils.Pointer("k8s-dqlite"), + ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, + ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, + ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, + ExtraNodeKubeProxyArgs: map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")}, + ExtraNodeKubeletArgs: map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")}, + ExtraNodeContainerdArgs: map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")}, + ExtraNodeK8sDqliteArgs: map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")}, }, }, { diff --git a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml index 71d7ed973..07068680c 100644 --- a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml +++ b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml @@ -31,3 +31,17 @@ k8s-dqlite-port: 9090 datastore-type: k8s-dqlite extra-sans: - custom.kubernetes +extra-node-kube-apiserver-args: + --extra-kube-apiserver-arg: extra-kube-apiserver-value +extra-node-kube-controller-manager-args: + --extra-kube-controller-manager-arg: extra-kube-controller-manager-value +extra-node-kube-scheduler-args: + --extra-kube-scheduler-arg: extra-kube-scheduler-value +extra-node-kube-proxy-args: + --extra-kube-proxy-arg: extra-kube-proxy-value +extra-node-kubelet-args: + --extra-kubelet-arg: extra-kubelet-value +extra-node-containerd-args: + --extra-containerd-arg: extra-containerd-value +extra-node-k8s-dqlite-args: + --extra-k8s-dqlite-arg: extra-k8s-dqlite-value diff --git a/src/k8s/pkg/k8sd/app/cluster_util.go b/src/k8s/pkg/k8sd/app/cluster_util.go index 2d5f55b5d..adb7842ad 100644 --- a/src/k8s/pkg/k8sd/app/cluster_util.go +++ b/src/k8s/pkg/k8sd/app/cluster_util.go @@ -3,12 +3,10 @@ package app import ( "context" "fmt" - "net" "path" "github.com/canonical/k8s/pkg/k8sd/pki" "github.com/canonical/k8s/pkg/k8sd/setup" - "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" "github.com/canonical/microcluster/state" @@ -35,29 +33,6 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki } -func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.ClusterConfig, nodeIP net.IP) error { - // Configure services - if err := setup.Containerd(snap, nil); err != nil { - return fmt.Errorf("failed to configure containerd: %w", err) - } - if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints()); err != nil { - return fmt.Errorf("failed to configure kubelet: %w", err) - } - if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR()); err != nil { - return fmt.Errorf("failed to configure kube-proxy: %w", err) - } - if err := setup.KubeControllerManager(snap); err != nil { - return fmt.Errorf("failed to configure kube-controller-manager: %w", err) - } - if err := setup.KubeScheduler(snap); err != nil { - return fmt.Errorf("failed to configure kube-scheduler: %w", err) - } - if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil { - return fmt.Errorf("failed to configure kube-apiserver: %w", err) - } - return nil -} - func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error { // Start services switch datastore { diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index 34033075c..ab90344dd 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -189,17 +189,17 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon } // Worker node services - if err := setup.Containerd(snap, nil); err != nil { + if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil { return fmt.Errorf("failed to configure containerd: %w", err) } - if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil { + if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil { return fmt.Errorf("failed to configure kubelet: %w", err) } - if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR); err != nil { + if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR, joinConfig.ExtraNodeKubeProxyArgs); err != nil { return fmt.Errorf("failed to configure kube-proxy: %w", err) } - if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil { - return fmt.Errorf("failed to configure kube-proxy: %w", err) + if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { + return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) } // TODO(berkayoz): remove the lock on cleanup @@ -344,7 +344,7 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot // Configure datastore switch cfg.Datastore.GetType() { case "k8s-dqlite": - if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil); err != nil { + if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil { return fmt.Errorf("failed to configure k8s-dqlite: %w", err) } case "external": @@ -353,8 +353,23 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot } // Configure services - if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil { - return fmt.Errorf("failed to configure services: %w", err) + if err := setup.Containerd(snap, nil, bootstrapConfig.ExtraNodeContainerdArgs); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) + } + if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) + } + if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) + } + if err := setup.KubeControllerManager(snap, bootstrapConfig.ExtraNodeKubeControllerManagerArgs); err != nil { + return fmt.Errorf("failed to configure kube-controller-manager: %w", err) + } + if err := setup.KubeScheduler(snap, bootstrapConfig.ExtraNodeKubeSchedulerArgs); err != nil { + return fmt.Errorf("failed to configure kube-scheduler: %w", err) + } + if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil { + return fmt.Errorf("failed to configure kube-apiserver: %w", err) } // Write cluster configuration to dqlite diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 2382525f0..38d56a30f 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -134,7 +134,7 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { } address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()) - if err := setup.K8sDqlite(snap, address, cluster); err != nil { + if err := setup.K8sDqlite(snap, address, cluster, joinConfig.ExtraNodeK8sDqliteArgs); err != nil { return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err) } case "external": @@ -143,8 +143,23 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { } // Configure services - if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil { - return fmt.Errorf("failed to configure services: %w", err) + if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) + } + if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) + } + if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), joinConfig.ExtraNodeKubeProxyArgs); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) + } + if err := setup.KubeControllerManager(snap, joinConfig.ExtraNodeKubeControllerManagerArgs); err != nil { + return fmt.Errorf("failed to configure kube-controller-manager: %w", err) + } + if err := setup.KubeScheduler(snap, joinConfig.ExtraNodeKubeSchedulerArgs); err != nil { + return fmt.Errorf("failed to configure kube-scheduler: %w", err) + } + if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil { + return fmt.Errorf("failed to configure kube-apiserver: %w", err) } if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil { diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go index 1e7abab33..e2476c186 100644 --- a/src/k8s/pkg/k8sd/setup/containerd.go +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -113,7 +113,7 @@ func containerdHostConfig(registry types.ContainerdRegistry) containerdHostsConf // Containerd configures configuration and arguments for containerd on the local node. // Optionally, a number of registry mirrors and auths can be configured. -func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error { +func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs map[string]*string) error { configToml, err := os.OpenFile(path.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to open config.toml: %w", err) @@ -138,6 +138,12 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error { return fmt.Errorf("failed to write arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "containerd", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } + cniBinary := path.Join(snap.CNIBinDir(), "cni") if err := utils.CopyFile(snap.CNIPluginsBinary(), cniBinary); err != nil { return fmt.Errorf("failed to copy cni plugin binary: %w", err) diff --git a/src/k8s/pkg/k8sd/setup/containerd_test.go b/src/k8s/pkg/k8sd/setup/containerd_test.go index 50b4c5556..9a898f1c7 100644 --- a/src/k8s/pkg/k8sd/setup/containerd_test.go +++ b/src/k8s/pkg/k8sd/setup/containerd_test.go @@ -12,6 +12,7 @@ import ( "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap/mock" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -40,7 +41,7 @@ func TestContainerd(t *testing.T) { }, } - g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) + g.Expect(setup.EnsureAllDirectories(s)).To(Succeed()) g.Expect(setup.Containerd(s, []types.ContainerdRegistry{ { Host: "docker.io", @@ -53,6 +54,11 @@ func TestContainerd(t *testing.T) { URLs: []string{"https://ghcr.mirror.internal"}, Token: "token", }, + }, map[string]*string{ + "--log-level": utils.Pointer("debug"), + "--metrics": utils.Pointer("true"), + "--address": nil, // This should trigger a delete + "--my-extra-arg": utils.Pointer("my-extra-val"), })).To(Succeed()) t.Run("Config", func(t *testing.T) { @@ -102,10 +108,12 @@ func TestContainerd(t *testing.T) { t.Run("Args", func(t *testing.T) { for key, expectedVal := range map[string]string{ - "--address": path.Join(dir, "containerd-run", "containerd.sock"), - "--config": path.Join(dir, "containerd", "config.toml"), - "--root": path.Join(dir, "containerd-root"), - "--state": path.Join(dir, "containerd-state"), + "--config": path.Join(dir, "containerd", "config.toml"), + "--root": path.Join(dir, "containerd-root"), + "--state": path.Join(dir, "containerd-state"), + "--log-level": "debug", + "--metrics": "true", + "--my-extra-arg": "my-extra-val", } { t.Run(key, func(t *testing.T) { g := NewWithT(t) @@ -114,6 +122,13 @@ func TestContainerd(t *testing.T) { g.Expect(val).To(Equal(expectedVal)) }) } + // --address was deleted by extraArgs + t.Run("--address", func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "containerd", "--address") + g.Expect(err).To(BeNil()) + g.Expect(val).To(BeZero()) + }) }) t.Run("Registries", func(t *testing.T) { diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go index a4c9f4f16..eccb1c540 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go @@ -10,7 +10,7 @@ import ( ) // K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy. -func K8sAPIServerProxy(snap snap.Snap, servers []string) error { +func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*string) error { configFile := path.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json") if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil { return fmt.Errorf("failed to write proxy configuration file: %w", err) @@ -24,5 +24,10 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string) error { return fmt.Errorf("failed to write arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go index 4bca4ddf5..3a9bf54b6 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go @@ -27,7 +27,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) - g.Expect(setup.K8sAPIServerProxy(s, nil)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).To(Succeed()) tests := []struct { key string @@ -45,6 +45,48 @@ func TestK8sApiServerProxy(t *testing.T) { g.Expect(tc.expectedVal).To(Equal(val)) }) } + + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-apiserver-proxy")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) + + extraArgs := map[string]*string{ + "--kubeconfig": utils.Pointer("overridden-kubelet.conf"), + "--listen": nil, // This should trigger a delete + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + g.Expect(setup.K8sAPIServerProxy(s, nil, extraArgs)).To(Succeed()) + + tests := []struct { + key string + expectedVal string + }{ + {key: "--endpoints", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json")}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "overridden-kubelet.conf")}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-apiserver-proxy", tc.key) + g.Expect(err).To(BeNil()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + // --listen was deleted by extraArgs + t.Run("--listen", func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-apiserver-proxy", "--listen") + g.Expect(err).To(BeNil()) + g.Expect(val).To(BeZero()) + }) + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-apiserver-proxy")) g.Expect(err).ToNot(HaveOccurred()) g.Expect(len(args)).To(Equal(len(tests))) @@ -56,7 +98,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceExtraConfigDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed()) }) t.Run("MissingServiceArgumentsDir", func(t *testing.T) { @@ -65,7 +107,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed()) }) t.Run("JSONFileContent", func(t *testing.T) { @@ -76,7 +118,7 @@ func TestK8sApiServerProxy(t *testing.T) { endpoints := []string{"192.168.0.1", "192.168.0.2", "192.168.0.3"} fileName := path.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json") - g.Expect(setup.K8sAPIServerProxy(s, endpoints)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, endpoints, nil)).To(Succeed()) b, err := os.ReadFile(fileName) g.Expect(err).NotTo(HaveOccurred()) diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go index 5e7ca6f3b..6ed7620bd 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go +++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go @@ -15,7 +15,7 @@ type k8sDqliteInit struct { Cluster []string `yaml:"Cluster,omitempty"` } -func K8sDqlite(snap snap.Snap, address string, cluster []string) error { +func K8sDqlite(snap snap.Snap, address string, cluster []string, extraArgs map[string]*string) error { b, err := yaml.Marshal(&k8sDqliteInit{Address: address, Cluster: cluster}) if err != nil { return fmt.Errorf("failed to create init.yaml file for address=%s cluster=%v: %w", address, cluster, err) @@ -31,5 +31,11 @@ func K8sDqlite(snap snap.Snap, address string, cluster []string) error { }, nil); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "k8s-dqlite", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go index 7f2dc733c..077f94e6e 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go +++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go @@ -28,7 +28,7 @@ func TestK8sDqlite(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sDqliteMock) // Call the K8sDqlite setup function with mock arguments - g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"})).To(BeNil()) + g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"}, nil)).To(BeNil()) // Ensure the K8sDqlite arguments file has the expected arguments and values tests := []struct { @@ -53,6 +53,51 @@ func TestK8sDqlite(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setK8sDqliteMock) + + extraArgs := map[string]*string{ + "--my-extra-arg": utils.Pointer("my-extra-val"), + "--listen": nil, + "--storage-dir": utils.Pointer("overridden-storage-dir"), + } + // Call the K8sDqlite setup function with mock arguments + g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"}, extraArgs)).To(BeNil()) + + // Ensure the K8sDqlite arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--storage-dir", expectedVal: "overridden-storage-dir"}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-dqlite", tc.key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(tc.expectedVal)) + }) + } + + // --listen was deleted by extraArgs + t.Run("--listen", func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "k8s-dqlite", "--listen") + g.Expect(err).To(BeNil()) + g.Expect(val).To(BeZero()) + }) + + // Ensure the K8sDqlite arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-dqlite")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + t.Run("YAMLFileContents", func(t *testing.T) { g := NewWithT(t) @@ -67,7 +112,7 @@ func TestK8sDqlite(t *testing.T) { "192.168.0.3:1234", } - g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", cluster)).To(BeNil()) + g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", cluster, nil)).To(BeNil()) b, err := os.ReadFile(path.Join(s.Mock.K8sDqliteStateDir, "init.yaml")) g.Expect(err).To(BeNil()) @@ -81,7 +126,7 @@ func TestK8sDqlite(t *testing.T) { s.Mock.K8sDqliteStateDir = "nonexistent" - g.Expect(setup.K8sDqlite(s, "", []string{})).ToNot(Succeed()) + g.Expect(setup.K8sDqlite(s, "", []string{}, nil)).ToNot(Succeed()) }) t.Run("MissingArgsDir", func(t *testing.T) { @@ -91,6 +136,6 @@ func TestK8sDqlite(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.K8sDqlite(s, "", []string{})).ToNot(Succeed()) + g.Expect(setup.K8sDqlite(s, "", []string{}, nil)).ToNot(Succeed()) }) } diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go index 09d797f27..b932311f5 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -47,7 +47,7 @@ var ( ) // KubeAPIServer configures kube-apiserver on the local node. -func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string) error { +func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string, extraArgs map[string]*string) error { authTokenWebhookConfigFile := path.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf") authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { @@ -105,5 +105,11 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, deleteArgs); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go index 6eede1f05..20e06adcb 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go @@ -36,7 +36,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -91,7 +91,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -132,13 +132,76 @@ func TestKubeAPIServer(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) + + extraArgs := map[string]*string{ + "--allow-privileged": nil, + "--secure-port": utils.Pointer("1337"), + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the KubeAPIServer setup function with mock arguments + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", extraArgs)).To(BeNil()) + + // Ensure the kube-apiserver arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--authentication-token-webhook-config-file", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "auth-token-webhook.conf")}, + {key: "--authorization-mode", expectedVal: "Node,RBAC"}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, + {key: "--enable-admission-plugins", expectedVal: "NodeRestriction"}, + {key: "--kubelet-certificate-authority", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--kubelet-client-certificate", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.crt")}, + {key: "--kubelet-client-key", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.key")}, + {key: "--kubelet-preferred-address-types", expectedVal: "InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP"}, + {key: "--secure-port", expectedVal: "1337"}, + {key: "--service-account-issuer", expectedVal: "https://kubernetes.default.svc"}, + {key: "--service-account-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")}, + {key: "--service-account-signing-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")}, + {key: "--service-cluster-ip-range", expectedVal: "10.0.0.0/24"}, + {key: "--tls-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver.crt")}, + {key: "--tls-cipher-suites", expectedVal: apiserverTLSCipherSuites}, + {key: "--tls-private-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver.key")}, + {key: "--etcd-servers", expectedVal: fmt.Sprintf("unix://%s", path.Join(s.Mock.K8sDqliteStateDir, "k8s-dqlite.sock"))}, + {key: "--requestheader-client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-ca.crt")}, + {key: "--requestheader-allowed-names", expectedVal: "front-proxy-client"}, + {key: "--requestheader-extra-headers-prefix", expectedVal: "X-Remote-Extra-"}, + {key: "--requestheader-group-headers", expectedVal: "X-Remote-Group"}, + {key: "--requestheader-username-headers", expectedVal: "X-Remote-User"}, + {key: "--proxy-client-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-client.crt")}, + {key: "--proxy-client-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-client.key")}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-apiserver", tc.key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(tc.expectedVal)) + }) + } + // Ensure that the allow-privileged argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-apiserver", "--allow-privileged") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kube-apiserver arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) t.Run("ArgsDualstack", func(t *testing.T) { g := NewWithT(t) s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--service-cluster-ip-range")).To(Equal("10.0.0.0/24,fd01::/64")) _, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -151,7 +214,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--etcd-servers")).To(Equal("datastoreurl1,datastoreurl2")) _, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -165,7 +228,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Attempt to configure kube-apiserver with an unsupported datastore - err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC") + err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC", nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(ContainSubstring("unsupported datastore"))) }) diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go index a76bb6643..5b3bba22c 100644 --- a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go +++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go @@ -10,7 +10,7 @@ import ( ) // KubeControllerManager configures kube-controller-manager on the local node. -func KubeControllerManager(snap snap.Snap) error { +func KubeControllerManager(snap snap.Snap, extraArgs map[string]*string) error { args := map[string]string{ "--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), "--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), @@ -30,5 +30,10 @@ func KubeControllerManager(snap snap.Snap) error { if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", args, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go index b5c44f800..e6878b2a1 100644 --- a/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go @@ -31,7 +31,7 @@ func TestKubeControllerManager(t *testing.T) { os.Create(path.Join(s.Mock.KubernetesPKIDir, "ca.key")) // Call the kube controller manager setup function - g.Expect(setup.KubeControllerManager(s)).To(BeNil()) + g.Expect(setup.KubeControllerManager(s, nil)).To(BeNil()) // Ensure the kube controller manager arguments file has the expected arguments and values tests := []struct { @@ -67,7 +67,7 @@ func TestKubeControllerManager(t *testing.T) { t.Run("MissingArgsDir", func(t *testing.T) { g := NewWithT(t) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeControllerManager(s)).ToNot(Succeed()) + g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed()) }) }) @@ -78,7 +78,7 @@ func TestKubeControllerManager(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeControllerManagerMock) // Call the kube controller manager setup function - g.Expect(setup.KubeControllerManager(s)).To(BeNil()) + g.Expect(setup.KubeControllerManager(s, nil)).To(BeNil()) // Ensure the kube controller manager arguments file has the expected arguments and values tests := []struct { @@ -112,7 +112,67 @@ func TestKubeControllerManager(t *testing.T) { t.Run("MissingArgsDir", func(t *testing.T) { g := NewWithT(t) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeControllerManager(s)).ToNot(Succeed()) + g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed()) + }) + }) + + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeControllerManagerMock) + + // Create ca.key so that cluster-signing-cert-file and cluster-signing-key-file are added to the arguments + os.Create(path.Join(s.Mock.KubernetesPKIDir, "ca.key")) + + extraArgs := map[string]*string{ + "--leader-elect-lease-duration": nil, + "--profiling": utils.Pointer("true"), + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the kube controller manager setup function + g.Expect(setup.KubeControllerManager(s, extraArgs)).To(BeNil()) + + // Ensure the kube controller manager arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--authentication-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")}, + {key: "--authorization-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")}, + {key: "--leader-elect-renew-deadline", expectedVal: "15s"}, + {key: "--profiling", expectedVal: "true"}, + {key: "--root-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--service-account-private-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")}, + {key: "--use-service-account-credentials", expectedVal: "true"}, + {key: "--cluster-signing-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--cluster-signing-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.key")}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-controller-manager", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the leader-elect-lease-duration argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-controller-manager", "--leader-elect-lease-duration") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kube controller manager arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-controller-manager")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + + t.Run("MissingArgsDir", func(t *testing.T) { + g := NewWithT(t) + s.Mock.ServiceArgumentsDir = "nonexistent" + g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed()) }) }) } diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy.go b/src/k8s/pkg/k8sd/setup/kube_proxy.go index a1f7a59f6..6731b2925 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy.go @@ -11,7 +11,7 @@ import ( ) // KubeProxy configures kube-proxy on the local node. -func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string) error { +func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string, extraArgs map[string]*string) error { serviceArgs := map[string]string{ "--cluster-cidr": podCIDR, "--healthz-bind-address": "127.0.0.1", @@ -33,5 +33,11 @@ func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR str if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", serviceArgs, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go index 17d8ac382..69c71d921 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go @@ -9,6 +9,7 @@ import ( "github.com/canonical/k8s/pkg/k8sd/setup" "github.com/canonical/k8s/pkg/snap/mock" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -30,11 +31,10 @@ func TestKubeProxy(t *testing.T) { g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) t.Run("Args", func(t *testing.T) { - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil()) for key, expectedVal := range map[string]string{ "--cluster-cidr": "10.1.0.0/16", - "--healthz-bind-address": "127.0.0.1", "--hostname-override": "myhostname", "--kubeconfig": path.Join(dir, "kubernetes", "proxy.conf"), "--profiling": "false", @@ -49,9 +49,38 @@ func TestKubeProxy(t *testing.T) { } }) + t.Run("WithExtraArgs", func(t *testing.T) { + extraArgs := map[string]*string{ + "--hostname-override": utils.Pointer("myoverriddenhostname"), + "--healthz-bind-address": nil, + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", extraArgs)).To(BeNil()) + + for key, expectedVal := range map[string]string{ + "--cluster-cidr": "10.1.0.0/16", + "--hostname-override": "myoverriddenhostname", + "--kubeconfig": path.Join(dir, "kubernetes", "proxy.conf"), + "--profiling": "false", + "--conntrack-max-per-core": "", + "--my-extra-arg": "my-extra-val", + } { + t.Run(key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-proxy", key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(expectedVal)) + }) + } + // Ensure that the healthz-bind-address argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-proxy", "--healthz-bind-address") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + }) + s.Mock.OnLXD = true t.Run("ArgsOnLXD", func(t *testing.T) { - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil()) for key, expectedVal := range map[string]string{ "--conntrack-max-per-core": "0", diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler.go b/src/k8s/pkg/k8sd/setup/kube_scheduler.go index 1cc6ae5b4..da151a0bc 100644 --- a/src/k8s/pkg/k8sd/setup/kube_scheduler.go +++ b/src/k8s/pkg/k8sd/setup/kube_scheduler.go @@ -9,7 +9,7 @@ import ( ) // KubeScheduler configures kube-scheduler on the local node. -func KubeScheduler(snap snap.Snap) error { +func KubeScheduler(snap snap.Snap, extraArgs map[string]*string) error { if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", map[string]string{ "--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), "--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), @@ -20,5 +20,10 @@ func KubeScheduler(snap snap.Snap) error { }, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go b/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go index f4e9f62b1..d478bd373 100644 --- a/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go @@ -26,7 +26,7 @@ func TestKubeScheduler(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeSchedulerMock) // Call the kube scheduler setup function - g.Expect(setup.KubeScheduler(s)).To(BeNil()) + g.Expect(setup.KubeScheduler(s, nil)).To(BeNil()) // Ensure the kube scheduler arguments file has the expected arguments and values tests := []struct { @@ -56,6 +56,53 @@ func TestKubeScheduler(t *testing.T) { }) + t.Run("WithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeSchedulerMock) + + extraArgs := map[string]*string{ + "--leader-elect-lease-duration": nil, + "--profiling": utils.Pointer("true"), + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the kube scheduler setup function + g.Expect(setup.KubeScheduler(s, extraArgs)).To(BeNil()) + + // Ensure the kube scheduler arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--authentication-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")}, + {key: "--authorization-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")}, + {key: "--leader-elect-renew-deadline", expectedVal: "15s"}, + {key: "--profiling", expectedVal: "true"}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kube-scheduler", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the leader-elect-lease-duration argument was deleted + val, err := snaputil.GetServiceArgument(s, "kube-scheduler", "--leader-elect-lease-duration") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kube scheduler arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-scheduler")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + + }) + t.Run("MissingArgsDir", func(t *testing.T) { g := NewWithT(t) @@ -63,6 +110,6 @@ func TestKubeScheduler(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeScheduler(s)).ToNot(Succeed()) + g.Expect(setup.KubeScheduler(s, nil)).ToNot(Succeed()) }) } diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go index 7ad85fcdf..bac5da7d2 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet.go +++ b/src/k8s/pkg/k8sd/setup/kubelet.go @@ -30,17 +30,17 @@ var kubeletWorkerLabels = []string{ } // KubeletControlPlane configures kubelet on a control plane node. -func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, registerWithTaints []string) error { - return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, registerWithTaints, append(kubeletControlPlaneLabels, kubeletWorkerLabels...)) +func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, registerWithTaints []string, extraArgs map[string]*string) error { + return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, registerWithTaints, append(kubeletControlPlaneLabels, kubeletWorkerLabels...), extraArgs) } // KubeletWorker configures kubelet on a worker node. -func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string) error { - return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels) +func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, extraArgs map[string]*string) error { + return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels, extraArgs) } // kubelet configures kubelet on the local node. -func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, taints []string, labels []string) error { +func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, taints []string, labels []string, extraArgs map[string]*string) error { args := map[string]string{ "--anonymous-auth": "false", "--authentication-token-webhook": "true", @@ -74,5 +74,11 @@ func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", args, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } + + // Apply extra arguments after the defaults, so they can override them. + updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", updateArgs, deleteArgs); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } return nil } diff --git a/src/k8s/pkg/k8sd/setup/kubelet_test.go b/src/k8s/pkg/k8sd/setup/kubelet_test.go index 59d11e980..76686491a 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet_test.go +++ b/src/k8s/pkg/k8sd/setup/kubelet_test.go @@ -46,7 +46,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet control plane setup function - g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).To(Succeed()) + g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, nil)).To(Succeed()) // Ensure the kubelet arguments file has the expected arguments and values tests := []struct { @@ -69,8 +69,8 @@ func TestKubelet(t *testing.T) { {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir}, {key: "--serialize-image-pulls", expectedVal: "false"}, {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites}, - {key: "--cloud-provider", expectedVal: "provider"}, {key: "--cluster-dns", expectedVal: "10.152.1.1"}, + {key: "--cloud-provider", expectedVal: "provider"}, {key: "--cluster-domain", expectedVal: "test-cluster.local"}, {key: "--node-ip", expectedVal: "192.168.0.1"}, } @@ -89,6 +89,67 @@ func TestKubelet(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("ControlPlaneWithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeletMock) + + extraArgs := map[string]*string{ + "--cluster-domain": utils.Pointer("override.local"), + "--cloud-provider": nil, // This should trigger a delete + "--my-extra-arg": utils.Pointer("my-extra-val"), + } + // Call the kubelet control plane setup function + g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, extraArgs)).To(Succeed()) + + // Ensure the kubelet arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--anonymous-auth", expectedVal: "false"}, + {key: "--authentication-token-webhook", expectedVal: "true"}, + {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, + {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, + {key: "--fail-swap-on", expectedVal: "false"}, + {key: "--hostname-override", expectedVal: "dev"}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")}, + {key: "--node-labels", expectedVal: expectedControlPlaneLabels}, + {key: "--read-only-port", expectedVal: "0"}, + {key: "--register-with-taints", expectedVal: ""}, + {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir}, + {key: "--serialize-image-pulls", expectedVal: "false"}, + {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites}, + {key: "--cluster-dns", expectedVal: "10.152.1.1"}, + // Overwritten by extraArgs + {key: "--cluster-domain", expectedVal: "override.local"}, + {key: "--node-ip", expectedVal: "192.168.0.1"}, + {key: "--my-extra-arg", expectedVal: "my-extra-val"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kubelet", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the cloud-provider argument was deleted + val, err := snaputil.GetServiceArgument(s, "kubelet", "--cloud-provider") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kubelet arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kubelet")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + t.Run("ControlPlaneArgsNoOptional", func(t *testing.T) { g := NewWithT(t) @@ -96,7 +157,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet control plane setup function - g.Expect(setup.KubeletControlPlane(s, "dev", nil, "", "", "", nil)).To(BeNil()) + g.Expect(setup.KubeletControlPlane(s, "dev", nil, "", "", "", nil, nil)).To(BeNil()) tests := []struct { key string @@ -141,7 +202,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet worker setup function - g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider")).To(BeNil()) + g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).To(BeNil()) // Ensure the kubelet arguments file has the expected arguments and values tests := []struct { @@ -184,6 +245,65 @@ func TestKubelet(t *testing.T) { g.Expect(len(args)).To(Equal(len(tests))) }) + t.Run("WorkerWithExtraArgs", func(t *testing.T) { + g := NewWithT(t) + + // Create a mock snap + s := mustSetupSnapAndDirectories(t, setKubeletMock) + + extraArgs := map[string]*string{ + "--cluster-domain": utils.Pointer("override.local"), + "--cloud-provider": nil, + } + + // Call the kubelet worker setup function + g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", extraArgs)).To(BeNil()) + + // Ensure the kubelet arguments file has the expected arguments and values + tests := []struct { + key string + expectedVal string + }{ + {key: "--anonymous-auth", expectedVal: "false"}, + {key: "--authentication-token-webhook", expectedVal: "true"}, + {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, + {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, + {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, + {key: "--fail-swap-on", expectedVal: "false"}, + {key: "--hostname-override", expectedVal: "dev"}, + {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")}, + {key: "--node-labels", expectedVal: expectedWorkerLabels}, + {key: "--read-only-port", expectedVal: "0"}, + {key: "--register-with-taints", expectedVal: ""}, + {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir}, + {key: "--serialize-image-pulls", expectedVal: "false"}, + {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites}, + {key: "--cluster-dns", expectedVal: "10.152.1.1"}, + {key: "--cluster-domain", expectedVal: "override.local"}, + {key: "--node-ip", expectedVal: "192.168.0.1"}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "kubelet", tc.key) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(tc.expectedVal).To(Equal(val)) + }) + } + + // Ensure that the cloud-provider argument was deleted + val, err := snaputil.GetServiceArgument(s, "kubelet", "--cloud-provider") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(val).To(BeZero()) + + // Ensure the kubelet arguments file has exactly the expected number of arguments + args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kubelet")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(args)).To(Equal(len(tests))) + }) + t.Run("WorkerArgsNoOptional", func(t *testing.T) { g := NewWithT(t) @@ -191,7 +311,7 @@ func TestKubelet(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeletMock) // Call the kubelet worker setup function - g.Expect(setup.KubeletWorker(s, "dev", nil, "", "", "")).To(BeNil()) + g.Expect(setup.KubeletWorker(s, "dev", nil, "", "", "", nil)).To(BeNil()) // Ensure the kubelet arguments file has the expected arguments and values tests := []struct { @@ -236,7 +356,7 @@ func TestKubelet(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).ToNot(Succeed()) + g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, nil)).ToNot(Succeed()) }) t.Run("WorkerNoArgsDir", func(t *testing.T) { @@ -245,6 +365,6 @@ func TestKubelet(t *testing.T) { s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider")).ToNot(Succeed()) + g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).ToNot(Succeed()) }) } diff --git a/src/k8s/pkg/snap/util/services.go b/src/k8s/pkg/snap/util/services.go index 56be97bf2..d7587207e 100644 --- a/src/k8s/pkg/snap/util/services.go +++ b/src/k8s/pkg/snap/util/services.go @@ -75,3 +75,20 @@ func StopK8sDqliteServices(ctx context.Context, snap snap.Snap) error { } return nil } + +// ServiceArgsFromMap processes a map of string pointers and categorizes them into update and delete lists. +// - If the value pointer is nil, it adds the argument name to the delete list. +// - If the value pointer is not nil, it adds the argument and its value to the update map. +func ServiceArgsFromMap(args map[string]*string) (map[string]string, []string) { + updateArgs := make(map[string]string) + deleteArgs := make([]string, 0) + + for arg, val := range args { + if val == nil { + deleteArgs = append(deleteArgs, arg) + } else { + updateArgs[arg] = *val + } + } + return updateArgs, deleteArgs +} diff --git a/src/k8s/pkg/snap/util/services_test.go b/src/k8s/pkg/snap/util/services_test.go index 203012d81..70ae0bc75 100644 --- a/src/k8s/pkg/snap/util/services_test.go +++ b/src/k8s/pkg/snap/util/services_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/canonical/k8s/pkg/snap/mock" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -108,3 +109,73 @@ func TestStopK8sDqliteServices(t *testing.T) { g.Expect(StopK8sDqliteServices(context.Background(), mock)).NotTo(Succeed()) }) } + +func TestServiceArgsFromMap(t *testing.T) { + tests := []struct { + name string + input map[string]*string + expected struct { + updateArgs map[string]string + deleteArgs []string + } + }{ + { + name: "NilValue", + input: map[string]*string{"arg1": nil}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{}, + deleteArgs: []string{"arg1"}, + }, + }, + { + name: "EmptyString", // Should be threated as normal string + input: map[string]*string{"arg1": utils.Pointer("")}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": ""}, + deleteArgs: []string{}, + }, + }, + { + name: "NonEmptyString", + input: map[string]*string{"arg1": utils.Pointer("value1")}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": "value1"}, + deleteArgs: []string{}, + }, + }, + { + name: "MixedValues", + input: map[string]*string{ + "arg1": utils.Pointer("value1"), + "arg2": utils.Pointer(""), + "arg3": nil, + }, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": "value1", "arg2": ""}, + deleteArgs: []string{"arg3"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + updateArgs, deleteArgs := ServiceArgsFromMap(tt.input) + g.Expect(updateArgs).To(Equal(tt.expected.updateArgs)) + g.Expect(deleteArgs).To(Equal(tt.expected.deleteArgs)) + }) + } +} From 687bb19cb1214ec8c84e21d1ea4c0654d6ca87a7 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Tue, 11 Jun 2024 13:38:58 +0200 Subject: [PATCH 2/6] Add bootstrap config test --- src/k8s/api/v1/bootstrap_config_test.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/k8s/api/v1/bootstrap_config_test.go b/src/k8s/api/v1/bootstrap_config_test.go index 097f12574..6e4654404 100644 --- a/src/k8s/api/v1/bootstrap_config_test.go +++ b/src/k8s/api/v1/bootstrap_config_test.go @@ -41,13 +41,20 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) { }, CloudProvider: utils.Pointer("external"), }, - PodCIDR: utils.Pointer("10.100.0.0/16"), - ServiceCIDR: utils.Pointer("10.200.0.0/16"), - DisableRBAC: utils.Pointer(false), - SecurePort: utils.Pointer(6443), - K8sDqlitePort: utils.Pointer(9090), - DatastoreType: utils.Pointer("k8s-dqlite"), - ExtraSANs: []string{"custom.kubernetes"}, + PodCIDR: utils.Pointer("10.100.0.0/16"), + ServiceCIDR: utils.Pointer("10.200.0.0/16"), + DisableRBAC: utils.Pointer(false), + SecurePort: utils.Pointer(6443), + K8sDqlitePort: utils.Pointer(9090), + DatastoreType: utils.Pointer("k8s-dqlite"), + ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, + ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, + ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, + ExtraNodeKubeProxyArgs: map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")}, + ExtraNodeKubeletArgs: map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")}, + ExtraNodeContainerdArgs: map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")}, + ExtraNodeK8sDqliteArgs: map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")}, } microclusterConfig, err := cfg.ToMicrocluster() From a1c04d3d214e42becc6183eab5a715e713a8d15e Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Tue, 11 Jun 2024 15:10:39 +0200 Subject: [PATCH 3/6] Add extra config files parameters --- src/k8s/api/v1/bootstrap_config.go | 3 + src/k8s/api/v1/bootstrap_config_test.go | 1 + src/k8s/api/v1/join_config.go | 6 ++ src/k8s/cmd/k8s/k8s_bootstrap_test.go | 1 + .../k8s/testdata/bootstrap-config-full.yaml | 2 + src/k8s/pkg/k8sd/app/hooks_bootstrap.go | 7 ++ src/k8s/pkg/k8sd/app/hooks_join.go | 4 + src/k8s/pkg/k8sd/setup/util_extra_files.go | 45 ++++++++++++ .../pkg/k8sd/setup/util_extra_files_test.go | 73 +++++++++++++++++++ .../integration/templates/bootstrap-all.yaml | 15 ---- .../templates/bootstrap-session.yaml | 33 +++++++++ tests/integration/tests/conftest.py | 5 +- tests/integration/tests/test_smoke.py | 22 ++++++ tests/integration/tests/test_util/util.py | 4 +- 14 files changed, 202 insertions(+), 19 deletions(-) create mode 100644 src/k8s/pkg/k8sd/setup/util_extra_files.go create mode 100644 src/k8s/pkg/k8sd/setup/util_extra_files_test.go delete mode 100644 tests/integration/templates/bootstrap-all.yaml create mode 100644 tests/integration/templates/bootstrap-session.yaml diff --git a/src/k8s/api/v1/bootstrap_config.go b/src/k8s/api/v1/bootstrap_config.go index 190f7d4e7..8efa4797c 100644 --- a/src/k8s/api/v1/bootstrap_config.go +++ b/src/k8s/api/v1/bootstrap_config.go @@ -55,6 +55,9 @@ type BootstrapConfig struct { KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d + ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"` + // Extra args to add to individual services (set any arg to null to delete) ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` diff --git a/src/k8s/api/v1/bootstrap_config_test.go b/src/k8s/api/v1/bootstrap_config_test.go index 6e4654404..400be832f 100644 --- a/src/k8s/api/v1/bootstrap_config_test.go +++ b/src/k8s/api/v1/bootstrap_config_test.go @@ -48,6 +48,7 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) { K8sDqlitePort: utils.Pointer(9090), DatastoreType: utils.Pointer("k8s-dqlite"), ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeConfigFiles: map[string]string{"extra-node-config-file": "file-content"}, ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, diff --git a/src/k8s/api/v1/join_config.go b/src/k8s/api/v1/join_config.go index f59b03cb1..7a69aaa52 100644 --- a/src/k8s/api/v1/join_config.go +++ b/src/k8s/api/v1/join_config.go @@ -26,6 +26,9 @@ type ControlPlaneNodeJoinConfig struct { KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d + ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"` + // Extra args to add to individual services (set any arg to null to delete) ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` @@ -44,6 +47,9 @@ type WorkerNodeJoinConfig struct { KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` + // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d + ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"` + // Extra args to add to individual services (set any arg to null to delete) ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` diff --git a/src/k8s/cmd/k8s/k8s_bootstrap_test.go b/src/k8s/cmd/k8s/k8s_bootstrap_test.go index de5eda72c..a10fc922d 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap_test.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap_test.go @@ -72,6 +72,7 @@ var testCases = []testCase{ K8sDqlitePort: utils.Pointer(9090), DatastoreType: utils.Pointer("k8s-dqlite"), ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeConfigFiles: map[string]string{"extra-node-config-file.yaml": "test-file-content"}, ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, diff --git a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml index 07068680c..8431de74d 100644 --- a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml +++ b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml @@ -31,6 +31,8 @@ k8s-dqlite-port: 9090 datastore-type: k8s-dqlite extra-sans: - custom.kubernetes +extra-node-config-files: + extra-node-config-file.yaml: test-file-content extra-node-kube-apiserver-args: --extra-kube-apiserver-arg: extra-kube-apiserver-value extra-node-kube-controller-manager-args: diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index ab90344dd..9599b8704 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -201,6 +201,9 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) } + if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil { + return fmt.Errorf("failed to write extra node config files: %w", err) + } // TODO(berkayoz): remove the lock on cleanup if err := snaputil.MarkAsWorkerNode(snap, true); err != nil { @@ -372,6 +375,10 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot return fmt.Errorf("failed to configure kube-apiserver: %w", err) } + if err := setup.ExtraNodeConfigFiles(snap, bootstrapConfig.ExtraNodeConfigFiles); err != nil { + return fmt.Errorf("failed to write extra node config files: %w", err) + } + // Write cluster configuration to dqlite if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { if _, err := database.SetClusterConfig(ctx, tx, cfg); err != nil { diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 38d56a30f..35d4d8e9a 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -162,6 +162,10 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { return fmt.Errorf("failed to configure kube-apiserver: %w", err) } + if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil { + return fmt.Errorf("failed to write extra node config files: %w", err) + } + if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil { return fmt.Errorf("failed to set snapd configuration from k8sd: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files.go b/src/k8s/pkg/k8sd/setup/util_extra_files.go new file mode 100644 index 000000000..5fb038655 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/util_extra_files.go @@ -0,0 +1,45 @@ +package setup + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/canonical/k8s/pkg/snap" +) + +// ExtraNodeConfigFiles writes the file contents to the specified filenames in the snap.ExtraFilesDir directory. +// The files are created with 0400 permissions and owned by root. +// The filenames must not contain any slashes to prevent path traversal. +func ExtraNodeConfigFiles(snap snap.Snap, files map[string]string) error { + for filename, content := range files { + if strings.Contains(filename, "/") { + return fmt.Errorf("file name %q must not contain any slashes (possible path-traversal prevented)", filename) + } + + filePath := path.Join(snap.ServiceExtraConfigDir(), filename) + // Create or truncate the file + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filePath, err) + } + defer file.Close() + + // Write the content to the file + _, err = file.WriteString(content) + if err != nil { + return fmt.Errorf("failed to write to file %s: %w", filePath, err) + } + + // Set file owner to root + if err := os.Chown(filePath, snap.UID(), snap.GID()); err != nil { + return fmt.Errorf("failed to change owner of file %s: %w", filePath, err) + } + + if err := os.Chmod(filePath, 0400); err != nil { + return fmt.Errorf("failed to change mode of file %s: %w", filePath, err) + } + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files_test.go b/src/k8s/pkg/k8sd/setup/util_extra_files_test.go new file mode 100644 index 000000000..ad4fa38bb --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/util_extra_files_test.go @@ -0,0 +1,73 @@ +package setup + +import ( + "os" + "path/filepath" + "testing" + + "github.com/canonical/k8s/pkg/snap/mock" + "github.com/onsi/gomega" +) + +func TestExtraNodeConfigFiles(t *testing.T) { + tests := []struct { + name string + files map[string]string + expectErr bool + errMessage string + }{ + { + name: "ValidFiles", + files: map[string]string{ + "config1": "content1", + "config2": "content2", + }, + expectErr: false, + }, + { + name: "InvalidFilename", + files: map[string]string{ + "invalid/config": "content", + }, + expectErr: true, + errMessage: "file name \"invalid/config\" must not contain any slashes", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + tmpDir := t.TempDir() + snap := &mock.Snap{ + Mock: mock.Mock{ + ServiceExtraConfigDir: tmpDir, + UID: os.Getuid(), + GID: os.Getgid(), + }, + } + + err := ExtraNodeConfigFiles(snap, tt.files) + if tt.expectErr { + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err.Error()).To(gomega.ContainSubstring(tt.errMessage)) + } else { + g.Expect(err).ToNot(gomega.HaveOccurred()) + + for filename, content := range tt.files { + filePath := filepath.Join(tmpDir, filename) + + // Verify the file exists + info, err := os.Stat(filePath) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(info.Mode().Perm()).To(gomega.Equal(os.FileMode(0400))) + + // Verify the file content + actualContent, err := os.ReadFile(filePath) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(string(actualContent)).To(gomega.Equal(content)) + } + } + }) + } +} diff --git a/tests/integration/templates/bootstrap-all.yaml b/tests/integration/templates/bootstrap-all.yaml deleted file mode 100644 index 68ad1fbb4..000000000 --- a/tests/integration/templates/bootstrap-all.yaml +++ /dev/null @@ -1,15 +0,0 @@ -cluster-config: - network: - enabled: true - dns: - enabled: true - ingress: - enabled: true - load-balancer: - enabled: true - local-storage: - enabled: true - gateway: - enabled: true - metrics-server: - enabled: true diff --git a/tests/integration/templates/bootstrap-session.yaml b/tests/integration/templates/bootstrap-session.yaml new file mode 100644 index 000000000..ca569664f --- /dev/null +++ b/tests/integration/templates/bootstrap-session.yaml @@ -0,0 +1,33 @@ +# Contains the bootstrap configuration for the session instance of the integration tests. +# The session instance persists over test runs and is used to speed-up the integration tests. +cluster-config: + network: + enabled: true + dns: + enabled: true + ingress: + enabled: true + load-balancer: + enabled: true + local-storage: + enabled: true + gateway: + enabled: true + metrics-server: + enabled: true +extra-node-config-files: + bootstrap-extra-file.yaml: extra-args-test-file-content +extra-node-kube-apiserver-args: + --request-timeout: 2m +extra-node-kube-controller-manager-args: + --leader-elect-retry-period: 3s +extra-node-kube-scheduler-args: + --authorization-webhook-cache-authorized-ttl: 11s +extra-node-kube-proxy-args: + --config-sync-period: 14m +extra-node-kubelet-args: + --authentication-token-webhook-cache-ttl: 3m +extra-node-containerd-args: + --log-level: debug +extra-node-k8s-dqlite-args: + --watch-storage-available-size-interval: 6s diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py index 49926c3cd..309cf982b 100644 --- a/tests/integration/tests/conftest.py +++ b/tests/integration/tests/conftest.py @@ -126,9 +126,10 @@ def session_instance( instance = h.new_instance() util.setup_k8s_snap(instance, snap_path) - bootstrap_config_path = "/home/ubuntu/bootstrap-all-features.yaml" + bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml" instance.send_file( - (config.MANIFESTS_DIR / "bootstrap-all.yaml").as_posix(), bootstrap_config_path + (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(), + bootstrap_config_path, ) instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path]) diff --git a/tests/integration/tests/test_smoke.py b/tests/integration/tests/test_smoke.py index 2e424904b..9bc9b4c85 100644 --- a/tests/integration/tests/test_smoke.py +++ b/tests/integration/tests/test_smoke.py @@ -17,3 +17,25 @@ def test_smoke(session_instance: harness.Instance): config = result.stdout.decode() assert len(config) > 0 assert "server: https://192.168.210.41" in config + + # Verify extra node configs + content = session_instance.exec( + ["cat", "/var/snap/k8s/common/args/conf.d/bootstrap-extra-file.yaml"], + capture_output=True, + ) + assert content.stdout.decode() == "extra-args-test-file-content" + + # For each service, verify that the extra arg was written to the args file. + for service, value in { + "kube-apiserver": "--request-timeout=2m", + "kube-controller-manager": "--leader-elect-retry-period=3s", + "kube-scheduler": "--authorization-webhook-cache-authorized-ttl=11s", + "kube-proxy": "--config-sync-period=14m", + "kubelet": "--authentication-token-webhook-cache-ttl=3m", + "containerd": "--log-level=debug", + "k8s-dqlite": "--watch-storage-available-size-interval=6s", + }.items(): + args = session_instance.exec( + ["cat", f"/var/snap/k8s/common/args/{service}"], capture_output=True + ) + assert value in args.stdout.decode() diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index c9a62b452..599390e20 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -223,8 +223,8 @@ def get_join_token( # Join an existing cluster. -def join_cluster(instance: harness.Instance, join_token: str): - instance.exec(["k8s", "join-cluster", join_token]) +def join_cluster(instance: harness.Instance, join_token: str, *args: str): + instance.exec(["k8s", "join-cluster", join_token, *args]) def get_default_cidr(instance: harness.Instance, instance_default_ip: str): From bbc707b17cf2e3bba81c1b019924b12ccbfc4617 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Thu, 13 Jun 2024 11:04:27 +0200 Subject: [PATCH 4/6] address PR comments --- src/k8s/pkg/k8sd/setup/containerd.go | 2 +- src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go | 3 +- src/k8s/pkg/k8sd/setup/k8s_dqlite.go | 3 +- src/k8s/pkg/k8sd/setup/kube_apiserver.go | 3 +- .../pkg/k8sd/setup/kube_controller_manager.go | 3 +- src/k8s/pkg/k8sd/setup/kube_proxy.go | 3 +- src/k8s/pkg/k8sd/setup/kube_scheduler.go | 3 +- src/k8s/pkg/k8sd/setup/kubelet.go | 3 +- src/k8s/pkg/k8sd/setup/util_extra_files.go | 14 +--- src/k8s/pkg/utils/services.go | 18 +++++ src/k8s/pkg/utils/services_test.go | 77 +++++++++++++++++++ .../templates/bootstrap-session.yaml | 16 ---- .../templates/bootstrap-smoke.yaml | 32 ++++++++ tests/integration/tests/test_smoke.py | 31 ++++++-- tests/integration/tests/test_util/util.py | 4 +- 15 files changed, 168 insertions(+), 47 deletions(-) create mode 100644 src/k8s/pkg/utils/services.go create mode 100644 src/k8s/pkg/utils/services_test.go create mode 100644 tests/integration/templates/bootstrap-smoke.yaml diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go index e2476c186..4b61a9fa4 100644 --- a/src/k8s/pkg/k8sd/setup/containerd.go +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -139,7 +139,7 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "containerd", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go index eccb1c540..f5683a6a7 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go @@ -7,6 +7,7 @@ import ( "github.com/canonical/k8s/pkg/proxy" "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" ) // K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy. @@ -25,7 +26,7 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*s } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go index 6ed7620bd..f9a95d6e6 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go +++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go @@ -7,6 +7,7 @@ import ( "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" "gopkg.in/yaml.v2" ) @@ -33,7 +34,7 @@ func K8sDqlite(snap snap.Snap, address string, cluster []string, extraArgs map[s } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "k8s-dqlite", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go index b932311f5..636b5f156 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -9,6 +9,7 @@ import ( "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" ) type apiserverAuthTokenWebhookTemplateConfig struct { @@ -107,7 +108,7 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go index 5b3bba22c..91f68fcd9 100644 --- a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go +++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go @@ -7,6 +7,7 @@ import ( "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" ) // KubeControllerManager configures kube-controller-manager on the local node. @@ -31,7 +32,7 @@ func KubeControllerManager(snap snap.Snap, extraArgs map[string]*string) error { return fmt.Errorf("failed to render arguments file: %w", err) } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy.go b/src/k8s/pkg/k8sd/setup/kube_proxy.go index 6731b2925..64da185d6 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy.go @@ -8,6 +8,7 @@ import ( "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" ) // KubeProxy configures kube-proxy on the local node. @@ -35,7 +36,7 @@ func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR str } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler.go b/src/k8s/pkg/k8sd/setup/kube_scheduler.go index da151a0bc..bc687af31 100644 --- a/src/k8s/pkg/k8sd/setup/kube_scheduler.go +++ b/src/k8s/pkg/k8sd/setup/kube_scheduler.go @@ -6,6 +6,7 @@ import ( "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" ) // KubeScheduler configures kube-scheduler on the local node. @@ -21,7 +22,7 @@ func KubeScheduler(snap snap.Snap, extraArgs map[string]*string) error { return fmt.Errorf("failed to render arguments file: %w", err) } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go index bac5da7d2..4563cb224 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet.go +++ b/src/k8s/pkg/k8sd/setup/kubelet.go @@ -8,6 +8,7 @@ import ( "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" ) var kubeletTLSCipherSuites = []string{ @@ -76,7 +77,7 @@ func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, } // Apply extra arguments after the defaults, so they can override them. - updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs) + updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs) if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", updateArgs, deleteArgs); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files.go b/src/k8s/pkg/k8sd/setup/util_extra_files.go index 5fb038655..e6fef0162 100644 --- a/src/k8s/pkg/k8sd/setup/util_extra_files.go +++ b/src/k8s/pkg/k8sd/setup/util_extra_files.go @@ -19,16 +19,8 @@ func ExtraNodeConfigFiles(snap snap.Snap, files map[string]string) error { } filePath := path.Join(snap.ServiceExtraConfigDir(), filename) - // Create or truncate the file - file, err := os.Create(filePath) - if err != nil { - return fmt.Errorf("failed to create file %s: %w", filePath, err) - } - defer file.Close() - // Write the content to the file - _, err = file.WriteString(content) - if err != nil { + if err := os.WriteFile(filePath, []byte(content), 0400); err != nil { return fmt.Errorf("failed to write to file %s: %w", filePath, err) } @@ -36,10 +28,6 @@ func ExtraNodeConfigFiles(snap snap.Snap, files map[string]string) error { if err := os.Chown(filePath, snap.UID(), snap.GID()); err != nil { return fmt.Errorf("failed to change owner of file %s: %w", filePath, err) } - - if err := os.Chmod(filePath, 0400); err != nil { - return fmt.Errorf("failed to change mode of file %s: %w", filePath, err) - } } return nil } diff --git a/src/k8s/pkg/utils/services.go b/src/k8s/pkg/utils/services.go new file mode 100644 index 000000000..596ee5425 --- /dev/null +++ b/src/k8s/pkg/utils/services.go @@ -0,0 +1,18 @@ +package utils + +// ServiceArgsFromMap processes a map of string pointers and categorizes them into update and delete lists. +// - If the value pointer is nil, it adds the argument name to the delete list. +// - If the value pointer is not nil, it adds the argument and its value to the update map. +func ServiceArgsFromMap(args map[string]*string) (map[string]string, []string) { + updateArgs := make(map[string]string) + deleteArgs := make([]string, 0) + + for arg, val := range args { + if val == nil { + deleteArgs = append(deleteArgs, arg) + } else { + updateArgs[arg] = *val + } + } + return updateArgs, deleteArgs +} diff --git a/src/k8s/pkg/utils/services_test.go b/src/k8s/pkg/utils/services_test.go new file mode 100644 index 000000000..8a18cbfc0 --- /dev/null +++ b/src/k8s/pkg/utils/services_test.go @@ -0,0 +1,77 @@ +package utils + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestServiceArgsFromMap(t *testing.T) { + tests := []struct { + name string + input map[string]*string + expected struct { + updateArgs map[string]string + deleteArgs []string + } + }{ + { + name: "NilValue", + input: map[string]*string{"arg1": nil}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{}, + deleteArgs: []string{"arg1"}, + }, + }, + { + name: "EmptyString", // Should be threated as normal string + input: map[string]*string{"arg1": Pointer("")}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": ""}, + deleteArgs: []string{}, + }, + }, + { + name: "NonEmptyString", + input: map[string]*string{"arg1": Pointer("value1")}, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": "value1"}, + deleteArgs: []string{}, + }, + }, + { + name: "MixedValues", + input: map[string]*string{ + "arg1": Pointer("value1"), + "arg2": Pointer(""), + "arg3": nil, + }, + expected: struct { + updateArgs map[string]string + deleteArgs []string + }{ + updateArgs: map[string]string{"arg1": "value1", "arg2": ""}, + deleteArgs: []string{"arg3"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + updateArgs, deleteArgs := ServiceArgsFromMap(tt.input) + g.Expect(updateArgs).To(Equal(tt.expected.updateArgs)) + g.Expect(deleteArgs).To(Equal(tt.expected.deleteArgs)) + }) + } +} diff --git a/tests/integration/templates/bootstrap-session.yaml b/tests/integration/templates/bootstrap-session.yaml index ca569664f..bdeebaece 100644 --- a/tests/integration/templates/bootstrap-session.yaml +++ b/tests/integration/templates/bootstrap-session.yaml @@ -15,19 +15,3 @@ cluster-config: enabled: true metrics-server: enabled: true -extra-node-config-files: - bootstrap-extra-file.yaml: extra-args-test-file-content -extra-node-kube-apiserver-args: - --request-timeout: 2m -extra-node-kube-controller-manager-args: - --leader-elect-retry-period: 3s -extra-node-kube-scheduler-args: - --authorization-webhook-cache-authorized-ttl: 11s -extra-node-kube-proxy-args: - --config-sync-period: 14m -extra-node-kubelet-args: - --authentication-token-webhook-cache-ttl: 3m -extra-node-containerd-args: - --log-level: debug -extra-node-k8s-dqlite-args: - --watch-storage-available-size-interval: 6s diff --git a/tests/integration/templates/bootstrap-smoke.yaml b/tests/integration/templates/bootstrap-smoke.yaml new file mode 100644 index 000000000..220a43c49 --- /dev/null +++ b/tests/integration/templates/bootstrap-smoke.yaml @@ -0,0 +1,32 @@ +# Contains the bootstrap configuration for the smoke test. +cluster-config: + network: + enabled: true + dns: + enabled: true + ingress: + enabled: true + load-balancer: + enabled: true + local-storage: + enabled: true + gateway: + enabled: true + metrics-server: + enabled: true +extra-node-config-files: + bootstrap-extra-file.yaml: extra-args-test-file-content +extra-node-kube-apiserver-args: + --request-timeout: 2m +extra-node-kube-controller-manager-args: + --leader-elect-retry-period: 3s +extra-node-kube-scheduler-args: + --authorization-webhook-cache-authorized-ttl: 11s +extra-node-kube-proxy-args: + --config-sync-period: 14m +extra-node-kubelet-args: + --authentication-token-webhook-cache-ttl: 3m +extra-node-containerd-args: + --log-level: debug +extra-node-k8s-dqlite-args: + --watch-storage-available-size-interval: 6s diff --git a/tests/integration/tests/test_smoke.py b/tests/integration/tests/test_smoke.py index 9bc9b4c85..7ff72158c 100644 --- a/tests/integration/tests/test_smoke.py +++ b/tests/integration/tests/test_smoke.py @@ -2,24 +2,39 @@ # Copyright 2024 Canonical, Ltd. # import logging +from typing import List -from test_util import harness +import pytest +from test_util import config, harness, util LOG = logging.getLogger(__name__) -def test_smoke(session_instance: harness.Instance): +@pytest.mark.node_count(1) +@pytest.mark.disable_k8s_bootstrapping() +def test_smoke(instances: List[harness.Instance]): + instance = instances[0] + + bootstrap_smoke_config_path = "/home/ubuntu/bootstrap-smoke.yaml" + instance.send_file( + (config.MANIFESTS_DIR / "bootstrap-smoke.yaml").as_posix(), + bootstrap_smoke_config_path, + ) + + instance.exec(["k8s", "bootstrap", "--file", bootstrap_smoke_config_path]) + util.wait_until_k8s_ready(instance, [instance]) + # Verify the functionality of the k8s config command during the smoke test. # It would be excessive to deploy a cluster solely for this purpose. - result = session_instance.exec( + result = instance.exec( "k8s config --server 192.168.210.41".split(), capture_output=True ) - config = result.stdout.decode() - assert len(config) > 0 - assert "server: https://192.168.210.41" in config + kubeconfig = result.stdout.decode() + assert len(kubeconfig) > 0 + assert "server: https://192.168.210.41" in kubeconfig # Verify extra node configs - content = session_instance.exec( + content = instance.exec( ["cat", "/var/snap/k8s/common/args/conf.d/bootstrap-extra-file.yaml"], capture_output=True, ) @@ -35,7 +50,7 @@ def test_smoke(session_instance: harness.Instance): "containerd": "--log-level=debug", "k8s-dqlite": "--watch-storage-available-size-interval=6s", }.items(): - args = session_instance.exec( + args = instance.exec( ["cat", f"/var/snap/k8s/common/args/{service}"], capture_output=True ) assert value in args.stdout.decode() diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 599390e20..c9a62b452 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -223,8 +223,8 @@ def get_join_token( # Join an existing cluster. -def join_cluster(instance: harness.Instance, join_token: str, *args: str): - instance.exec(["k8s", "join-cluster", join_token, *args]) +def join_cluster(instance: harness.Instance, join_token: str): + instance.exec(["k8s", "join-cluster", join_token]) def get_default_cidr(instance: harness.Instance, instance_default_ip: str): From a3426b87c9cfd9ff3ba938fd11856d4e2382da05 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Thu, 13 Jun 2024 12:25:22 +0200 Subject: [PATCH 5/6] update docs --- .../reference/bootstrap-config-reference.md | 105 +++++++++++++++++- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/docs/src/snap/reference/bootstrap-config-reference.md b/docs/src/snap/reference/bootstrap-config-reference.md index 7109d5aa6..8b506b736 100644 --- a/docs/src/snap/reference/bootstrap-config-reference.md +++ b/docs/src/snap/reference/bootstrap-config-reference.md @@ -37,6 +37,7 @@ Determines if the feature should be enabled. If omitted defaults to `true` #### cluster-config.dns.cluster-domain + **Type:** `string`
**Required:** `No`
@@ -59,7 +60,8 @@ Can be used to point to an external dns server when feature is disabled. **Type:** `list[string]`
**Required:** `No`
-Sets the upstream nameservers used to forward queries for out-of-cluster endpoints. +Sets the upstream nameservers used to forward queries for out-of-cluster +endpoints. If omitted defaults to `/etc/resolv.conf` and uses the nameservers of the node. @@ -83,7 +85,8 @@ If omitted defaults to `false` **Type:** `string`
**Required:** `No`
-Sets the name of the secret to be used for providing default encryption to ingresses. +Sets the name of the secret to be used for providing default encryption to +ingresses. Ingresses can specify another TLS secret in their resource definitions, in which case the default secret won't be used. @@ -117,7 +120,8 @@ If omitted defaults to `false` **Type:** `list[string]`
**Required:** `No`
-Sets the CIDRs used for assigning IP addresses to Kubernetes services with type `LoadBalancer`. +Sets the CIDRs used for assigning IP addresses to Kubernetes services with type +`LoadBalancer`. #### cluster-config.load-balancer.l2-mode @@ -316,7 +320,8 @@ The CA certificate to be used when communicating with the external datastore. **Type:** `string`
**Required:** `No`
-The client certificate to be used when communicating with the external datastore. +The client certificate to be used when communicating with the external +datastore. ### datastore-client-key @@ -386,7 +391,8 @@ If omitted defaults to an auto generated key. **Type:** `string`
**Required:** `No`
-The client certificate to be used by kubelet for communicating with the kube-apiserver. +The client certificate to be used by kubelet for communicating with the +kube-apiserver. If omitted defaults to an auto generated certificate. ### apiserver-kubelet-client-key @@ -437,6 +443,79 @@ If omitted defaults to an auto generated certificate. The key to be used for the kubelet. If omitted defaults to an auto generated key. +### extra-node-config-files: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional files that are uploaded `/var/snap/k8s/common/args/conf.d/` +to a node on bootstrap. These files can them be references by Kubernetes +service arguments. +The format is `map[]`. + +### extra-node-kube-apiserver-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to the `kube-apiserver` only for that +specific node. Overwrites default configuration. A parameter that is explicitly +set to `null` is deleted. The format is `map[<--flag-name>]`. + +### extra-node-kube-controller-manager-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to the `kube-controller-manager` only for +that specific node. Overwrites default configuration. A parameter that is +explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. + +### extra-node-kube-scheduler-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to the `kube-scheduler` only for that +specific node. Overwrites default configuration. A parameter that is explicitly +set to `null` is deleted. The format is `map[<--flag-name>]`. + +### extra-node-kube-proxy-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to the `kube-proxy` only for that +specific node. Overwrites default configuration. A parameter that is explicitly +set to `null` is deleted. The format is `map[<--flag-name>]`. + +### extra-node-kubelet-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to the `kubelet` only for that +specific node. Overwrites default configuration. A parameter that is explicitly +set to `null` is deleted. The format is `map[<--flag-name>]`. + +### extra-node-containerd-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to `containerd` only for that +specific node. Overwrites default configuration. A parameter that is explicitly +set to `null` is deleted. The format is `map[<--flag-name>]`. + +### extra-node-k8s-dqlite-args: + +**Type:** `map[string]string`
+**Required:** `No`
+ +Additional arguments that are passed to `k8s-dqlite` only for that +specific node. Overwrites default configuration. A parameter that is explicitly +set to `null` is deleted. The format is `map[<--flag-name>]`. + ## Example The following example configures and enables certain features, sets an external @@ -478,4 +557,20 @@ k8s-dqlite-port: 9090 datastore-type: k8s-dqlite extra-sans: - custom.kubernetes +extra-node-config-files: + bootstrap-extra-file.yaml: extra-args-test-file-content +extra-node-kube-apiserver-args: + --request-timeout: 2m +extra-node-kube-controller-manager-args: + --leader-elect-retry-period: 3s +extra-node-kube-scheduler-args: + --authorization-webhook-cache-authorized-ttl: 11s +extra-node-kube-proxy-args: + --config-sync-period: 14m +extra-node-kubelet-args: + --authentication-token-webhook-cache-ttl: 3m +extra-node-containerd-args: + --log-level: debug +extra-node-k8s-dqlite-args: + --watch-storage-available-size-interval: 6s ``` From cc8342bee76b1d48ba9f3536adb02ee79fb1e7fd Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Thu, 13 Jun 2024 15:03:57 +0200 Subject: [PATCH 6/6] minor docs update --- .../snap/reference/bootstrap-config-reference.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/snap/reference/bootstrap-config-reference.md b/docs/src/snap/reference/bootstrap-config-reference.md index 8b506b736..047622c5d 100644 --- a/docs/src/snap/reference/bootstrap-config-reference.md +++ b/docs/src/snap/reference/bootstrap-config-reference.md @@ -443,7 +443,7 @@ If omitted defaults to an auto generated certificate. The key to be used for the kubelet. If omitted defaults to an auto generated key. -### extra-node-config-files: +### extra-node-config-files **Type:** `map[string]string`
**Required:** `No`
@@ -453,7 +453,7 @@ to a node on bootstrap. These files can them be references by Kubernetes service arguments. The format is `map[]`. -### extra-node-kube-apiserver-args: +### extra-node-kube-apiserver-args **Type:** `map[string]string`
**Required:** `No`
@@ -462,7 +462,7 @@ Additional arguments that are passed to the `kube-apiserver` only for that specific node. Overwrites default configuration. A parameter that is explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. -### extra-node-kube-controller-manager-args: +### extra-node-kube-controller-manager-args **Type:** `map[string]string`
**Required:** `No`
@@ -471,7 +471,7 @@ Additional arguments that are passed to the `kube-controller-manager` only for that specific node. Overwrites default configuration. A parameter that is explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. -### extra-node-kube-scheduler-args: +### extra-node-kube-scheduler-args **Type:** `map[string]string`
**Required:** `No`
@@ -480,7 +480,7 @@ Additional arguments that are passed to the `kube-scheduler` only for that specific node. Overwrites default configuration. A parameter that is explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. -### extra-node-kube-proxy-args: +### extra-node-kube-proxy-args **Type:** `map[string]string`
**Required:** `No`
@@ -489,7 +489,7 @@ Additional arguments that are passed to the `kube-proxy` only for that specific node. Overwrites default configuration. A parameter that is explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. -### extra-node-kubelet-args: +### extra-node-kubelet-args **Type:** `map[string]string`
**Required:** `No`
@@ -498,7 +498,7 @@ Additional arguments that are passed to the `kubelet` only for that specific node. Overwrites default configuration. A parameter that is explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. -### extra-node-containerd-args: +### extra-node-containerd-args **Type:** `map[string]string`
**Required:** `No`
@@ -507,7 +507,7 @@ Additional arguments that are passed to `containerd` only for that specific node. Overwrites default configuration. A parameter that is explicitly set to `null` is deleted. The format is `map[<--flag-name>]`. -### extra-node-k8s-dqlite-args: +### extra-node-k8s-dqlite-args **Type:** `map[string]string`
**Required:** `No`