Add extra arguments for each service

src/k8s/api/v1/bootstrap_config.go

@@ -54,6 +54,15 @@ type BootstrapConfig struct {
 	KubeletKey        *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+	ExtraNodeKubeSchedulerArgs         map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+	ExtraNodeKubeProxyArgs             map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs               map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs            map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
 func (b *BootstrapConfig) GetDatastoreType() string        { return getField(b.DatastoreType) }

src/k8s/api/v1/join_config.go

@@ -25,6 +25,15 @@ type ControlPlaneNodeJoinConfig struct {
 	KubeletKey        *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+	ExtraNodeKubeSchedulerArgs         map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+	ExtraNodeKubeProxyArgs             map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs               map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs            map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
 type WorkerNodeJoinConfig struct {
@@ -34,6 +43,12 @@ type WorkerNodeJoinConfig struct {
 	KubeletClientKey    *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
 	KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
 	KubeProxyClientKey  *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeProxyArgs         map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs           map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs        map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"`
 }
 
 func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string {

src/k8s/cmd/k8s/k8s_bootstrap_test.go

@@ -64,14 +64,21 @@ var testCases = []testCase{
 				},
 				CloudProvider: utils.Pointer("external"),
 			},
-			ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
-			PodCIDR:            utils.Pointer("10.100.0.0/16"),
-			ServiceCIDR:        utils.Pointer("10.200.0.0/16"),
-			DisableRBAC:        utils.Pointer(false),
-			SecurePort:         utils.Pointer(6443),
-			K8sDqlitePort:      utils.Pointer(9090),
-			DatastoreType:      utils.Pointer("k8s-dqlite"),
-			ExtraSANs:          []string{"custom.kubernetes"},
+			ControlPlaneTaints:                 []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
+			PodCIDR:                            utils.Pointer("10.100.0.0/16"),
+			ServiceCIDR:                        utils.Pointer("10.200.0.0/16"),
+			DisableRBAC:                        utils.Pointer(false),
+			SecurePort:                         utils.Pointer(6443),
+			K8sDqlitePort:                      utils.Pointer(9090),
+			DatastoreType:                      utils.Pointer("k8s-dqlite"),
+			ExtraSANs:                          []string{"custom.kubernetes"},
+			ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+			ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+			ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+			ExtraNodeKubeProxyArgs:             map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+			ExtraNodeKubeletArgs:               map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+			ExtraNodeContainerdArgs:            map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+			ExtraNodeK8sDqliteArgs:             map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
 		},
 	},
 	{

src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml

@@ -31,3 +31,17 @@ k8s-dqlite-port: 9090
 datastore-type: k8s-dqlite
 extra-sans:
 - custom.kubernetes
+extra-node-kube-apiserver-args:
+  --extra-kube-apiserver-arg: extra-kube-apiserver-value
+extra-node-kube-controller-manager-args:
+  --extra-kube-controller-manager-arg: extra-kube-controller-manager-value
+extra-node-kube-scheduler-args:
+  --extra-kube-scheduler-arg: extra-kube-scheduler-value
+extra-node-kube-proxy-args:
+  --extra-kube-proxy-arg: extra-kube-proxy-value
+extra-node-kubelet-args:
+  --extra-kubelet-arg: extra-kubelet-value
+extra-node-containerd-args:
+  --extra-containerd-arg: extra-containerd-value
+extra-node-k8s-dqlite-args:
+  --extra-k8s-dqlite-arg: extra-k8s-dqlite-value

src/k8s/pkg/k8sd/app/cluster_util.go

@@ -3,12 +3,10 @@ package app
 import (
 	"context"
 	"fmt"
-	"net"
 	"path"
 
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
-	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
 	"github.com/canonical/microcluster/state"
@@ -35,29 +33,6 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki
 
 }
 
-func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.ClusterConfig, nodeIP net.IP) error {
-	// Configure services
-	if err := setup.Containerd(snap, nil); err != nil {
-		return fmt.Errorf("failed to configure containerd: %w", err)
-	}
-	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints()); err != nil {
-		return fmt.Errorf("failed to configure kubelet: %w", err)
-	}
-	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR()); err != nil {
-		return fmt.Errorf("failed to configure kube-proxy: %w", err)
-	}
-	if err := setup.KubeControllerManager(snap); err != nil {
-		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
-	}
-	if err := setup.KubeScheduler(snap); err != nil {
-		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
-	}
-	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil {
-		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
-	}
-	return nil
-}
-
 func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error {
 	// Start services
 	switch datastore {

src/k8s/pkg/k8sd/app/hooks_bootstrap.go

@@ -189,17 +189,17 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon
 	}
 
 	// Worker node services
-	if err := setup.Containerd(snap, nil); err != nil {
+	if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
 		return fmt.Errorf("failed to configure containerd: %w", err)
 	}
-	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil {
+	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil {
 		return fmt.Errorf("failed to configure kubelet: %w", err)
 	}
-	if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR); err != nil {
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR, joinConfig.ExtraNodeKubeProxyArgs); err != nil {
 		return fmt.Errorf("failed to configure kube-proxy: %w", err)
 	}
-	if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil {
-		return fmt.Errorf("failed to configure kube-proxy: %w", err)
+	if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil {
+		return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err)
 	}
 
 	// TODO(berkayoz): remove the lock on cleanup
@@ -344,7 +344,7 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 	// Configure datastore
 	switch cfg.Datastore.GetType() {
 	case "k8s-dqlite":
-		if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil); err != nil {
+		if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil {
 			return fmt.Errorf("failed to configure k8s-dqlite: %w", err)
 		}
 	case "external":
@@ -353,8 +353,23 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 	}
 
 	// Configure services
-	if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil {
-		return fmt.Errorf("failed to configure services: %w", err)
+	if err := setup.Containerd(snap, nil, bootstrapConfig.ExtraNodeContainerdArgs); err != nil {
+		return fmt.Errorf("failed to configure containerd: %w", err)
+	}
+	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil {
+		return fmt.Errorf("failed to configure kubelet: %w", err)
+	}
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-proxy: %w", err)
+	}
+	if err := setup.KubeControllerManager(snap, bootstrapConfig.ExtraNodeKubeControllerManagerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
+	}
+	if err := setup.KubeScheduler(snap, bootstrapConfig.ExtraNodeKubeSchedulerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
+	}
+	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
 	}
 
 	// Write cluster configuration to dqlite

src/k8s/pkg/k8sd/app/hooks_join.go

@@ -134,7 +134,7 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
 		}
 
 		address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort())
-		if err := setup.K8sDqlite(snap, address, cluster); err != nil {
+		if err := setup.K8sDqlite(snap, address, cluster, joinConfig.ExtraNodeK8sDqliteArgs); err != nil {
 			return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err)
 		}
 	case "external":
@@ -143,8 +143,23 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
 	}
 
 	// Configure services
-	if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil {
-		return fmt.Errorf("failed to configure services: %w", err)
+	if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
+		return fmt.Errorf("failed to configure containerd: %w", err)
+	}
+	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil {
+		return fmt.Errorf("failed to configure kubelet: %w", err)
+	}
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), joinConfig.ExtraNodeKubeProxyArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-proxy: %w", err)
+	}
+	if err := setup.KubeControllerManager(snap, joinConfig.ExtraNodeKubeControllerManagerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
+	}
+	if err := setup.KubeScheduler(snap, joinConfig.ExtraNodeKubeSchedulerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
+	}
+	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
 	}
 
 	if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil {

src/k8s/pkg/k8sd/setup/containerd.go

@@ -113,7 +113,7 @@ func containerdHostConfig(registry types.ContainerdRegistry) containerdHostsConf
 
 // Containerd configures configuration and arguments for containerd on the local node.
 // Optionally, a number of registry mirrors and auths can be configured.
-func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error {
+func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs map[string]*string) error {
 	configToml, err := os.OpenFile(path.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to open config.toml: %w", err)
@@ -138,6 +138,12 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error {
 		return fmt.Errorf("failed to write arguments file: %w", err)
 	}
 
+	// Apply extra arguments after the defaults, so they can override them.
+	updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs)
+	if _, err := snaputil.UpdateServiceArguments(snap, "containerd", updateArgs, deleteArgs); err != nil {
+		return fmt.Errorf("failed to write arguments file: %w", err)
+	}
+
 	cniBinary := path.Join(snap.CNIBinDir(), "cni")
 	if err := utils.CopyFile(snap.CNIPluginsBinary(), cniBinary); err != nil {
 		return fmt.Errorf("failed to copy cni plugin binary: %w", err)

src/k8s/pkg/k8sd/setup/containerd_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap/mock"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
+	"github.com/canonical/k8s/pkg/utils"
 	. "github.com/onsi/gomega"
 )
 
@@ -40,7 +41,7 @@ func TestContainerd(t *testing.T) {
 		},
 	}
 
-	g.Expect(setup.EnsureAllDirectories(s)).To(BeNil())
+	g.Expect(setup.EnsureAllDirectories(s)).To(Succeed())
 	g.Expect(setup.Containerd(s, []types.ContainerdRegistry{
 		{
 			Host:     "docker.io",
@@ -53,6 +54,11 @@ func TestContainerd(t *testing.T) {
 			URLs:  []string{"https://ghcr.mirror.internal"},
 			Token: "token",
 		},
+	}, map[string]*string{
+		"--log-level":    utils.Pointer("debug"),
+		"--metrics":      utils.Pointer("true"),
+		"--address":      nil, // This should trigger a delete
+		"--my-extra-arg": utils.Pointer("my-extra-val"),
 	})).To(Succeed())
 
 	t.Run("Config", func(t *testing.T) {
@@ -102,10 +108,12 @@ func TestContainerd(t *testing.T) {
 
 	t.Run("Args", func(t *testing.T) {
 		for key, expectedVal := range map[string]string{
-			"--address": path.Join(dir, "containerd-run", "containerd.sock"),
-			"--config":  path.Join(dir, "containerd", "config.toml"),
-			"--root":    path.Join(dir, "containerd-root"),
-			"--state":   path.Join(dir, "containerd-state"),
+			"--config":       path.Join(dir, "containerd", "config.toml"),
+			"--root":         path.Join(dir, "containerd-root"),
+			"--state":        path.Join(dir, "containerd-state"),
+			"--log-level":    "debug",
+			"--metrics":      "true",
+			"--my-extra-arg": "my-extra-val",
 		} {
 			t.Run(key, func(t *testing.T) {
 				g := NewWithT(t)
@@ -114,6 +122,13 @@ func TestContainerd(t *testing.T) {
 				g.Expect(val).To(Equal(expectedVal))
 			})
 		}
+		// --address was deleted by extraArgs
+		t.Run("--address", func(t *testing.T) {
+			g := NewWithT(t)
+			val, err := snaputil.GetServiceArgument(s, "containerd", "--address")
+			g.Expect(err).To(BeNil())
+			g.Expect(val).To(BeZero())
+		})
 	})
 
 	t.Run("Registries", func(t *testing.T) {

src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go

@@ -10,7 +10,7 @@ import (
 )
 
 // K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy.
-func K8sAPIServerProxy(snap snap.Snap, servers []string) error {
+func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*string) error {
 	configFile := path.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json")
 	if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil {
 		return fmt.Errorf("failed to write proxy configuration file: %w", err)
@@ -24,5 +24,10 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string) error {
 		return fmt.Errorf("failed to write arguments file: %w", err)
 	}
 
+	// Apply extra arguments after the defaults, so they can override them.
+	updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs)
+	if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", updateArgs, deleteArgs); err != nil {
+		return fmt.Errorf("failed to write arguments file: %w", err)
+	}
 	return nil
 }