Add extra arguments for each k8sd service

src/k8s/api/v1/bootstrap_config.go

@@ -54,6 +54,18 @@ type BootstrapConfig struct {
 	KubeletKey        *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+	ExtraNodeKubeSchedulerArgs         map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+	ExtraNodeKubeProxyArgs             map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs               map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs            map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
 func (b *BootstrapConfig) GetDatastoreType() string        { return getField(b.DatastoreType) }

src/k8s/api/v1/bootstrap_config_test.go

@@ -41,13 +41,21 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) {
 			},
 			CloudProvider: utils.Pointer("external"),
 		},
-		PodCIDR:       utils.Pointer("10.100.0.0/16"),
-		ServiceCIDR:   utils.Pointer("10.200.0.0/16"),
-		DisableRBAC:   utils.Pointer(false),
-		SecurePort:    utils.Pointer(6443),
-		K8sDqlitePort: utils.Pointer(9090),
-		DatastoreType: utils.Pointer("k8s-dqlite"),
-		ExtraSANs:     []string{"custom.kubernetes"},
+		PodCIDR:                            utils.Pointer("10.100.0.0/16"),
+		ServiceCIDR:                        utils.Pointer("10.200.0.0/16"),
+		DisableRBAC:                        utils.Pointer(false),
+		SecurePort:                         utils.Pointer(6443),
+		K8sDqlitePort:                      utils.Pointer(9090),
+		DatastoreType:                      utils.Pointer("k8s-dqlite"),
+		ExtraSANs:                          []string{"custom.kubernetes"},
+		ExtraNodeConfigFiles:               map[string]string{"extra-node-config-file": "file-content"},
+		ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+		ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+		ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+		ExtraNodeKubeProxyArgs:             map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+		ExtraNodeKubeletArgs:               map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+		ExtraNodeContainerdArgs:            map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+		ExtraNodeK8sDqliteArgs:             map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
 	}
 
 	microclusterConfig, err := cfg.ToMicrocluster()

src/k8s/api/v1/join_config.go

@@ -25,6 +25,18 @@ type ControlPlaneNodeJoinConfig struct {
 	KubeletKey        *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+	ExtraNodeKubeSchedulerArgs         map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+	ExtraNodeKubeProxyArgs             map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs               map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs            map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
 type WorkerNodeJoinConfig struct {
@@ -34,6 +46,15 @@ type WorkerNodeJoinConfig struct {
 	KubeletClientKey    *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
 	KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
 	KubeProxyClientKey  *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
+
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeProxyArgs         map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs           map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs        map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"`
 }
 
 func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string {

src/k8s/cmd/k8s/k8s_bootstrap_test.go

@@ -64,14 +64,22 @@ var testCases = []testCase{
 				},
 				CloudProvider: utils.Pointer("external"),
 			},
-			ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
-			PodCIDR:            utils.Pointer("10.100.0.0/16"),
-			ServiceCIDR:        utils.Pointer("10.200.0.0/16"),
-			DisableRBAC:        utils.Pointer(false),
-			SecurePort:         utils.Pointer(6443),
-			K8sDqlitePort:      utils.Pointer(9090),
-			DatastoreType:      utils.Pointer("k8s-dqlite"),
-			ExtraSANs:          []string{"custom.kubernetes"},
+			ControlPlaneTaints:                 []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
+			PodCIDR:                            utils.Pointer("10.100.0.0/16"),
+			ServiceCIDR:                        utils.Pointer("10.200.0.0/16"),
+			DisableRBAC:                        utils.Pointer(false),
+			SecurePort:                         utils.Pointer(6443),
+			K8sDqlitePort:                      utils.Pointer(9090),
+			DatastoreType:                      utils.Pointer("k8s-dqlite"),
+			ExtraSANs:                          []string{"custom.kubernetes"},
+			ExtraNodeConfigFiles:               map[string]string{"extra-node-config-file.yaml": "test-file-content"},
+			ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+			ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+			ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+			ExtraNodeKubeProxyArgs:             map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+			ExtraNodeKubeletArgs:               map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+			ExtraNodeContainerdArgs:            map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+			ExtraNodeK8sDqliteArgs:             map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
 		},
 	},
 	{

src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml

@@ -31,3 +31,19 @@ k8s-dqlite-port: 9090
 datastore-type: k8s-dqlite
 extra-sans:
 - custom.kubernetes
+extra-node-config-files:
+  extra-node-config-file.yaml: test-file-content
+extra-node-kube-apiserver-args:
+  --extra-kube-apiserver-arg: extra-kube-apiserver-value
+extra-node-kube-controller-manager-args:
+  --extra-kube-controller-manager-arg: extra-kube-controller-manager-value
+extra-node-kube-scheduler-args:
+  --extra-kube-scheduler-arg: extra-kube-scheduler-value
+extra-node-kube-proxy-args:
+  --extra-kube-proxy-arg: extra-kube-proxy-value
+extra-node-kubelet-args:
+  --extra-kubelet-arg: extra-kubelet-value
+extra-node-containerd-args:
+  --extra-containerd-arg: extra-containerd-value
+extra-node-k8s-dqlite-args:
+  --extra-k8s-dqlite-arg: extra-k8s-dqlite-value

src/k8s/pkg/k8sd/app/cluster_util.go

@@ -3,12 +3,10 @@ package app
 import (
 	"context"
 	"fmt"
-	"net"
 	"path"
 
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
-	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
 	"github.com/canonical/microcluster/state"
@@ -35,29 +33,6 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki
 
 }
 
-func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.ClusterConfig, nodeIP net.IP) error {
-	// Configure services
-	if err := setup.Containerd(snap, nil); err != nil {
-		return fmt.Errorf("failed to configure containerd: %w", err)
-	}
-	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints()); err != nil {
-		return fmt.Errorf("failed to configure kubelet: %w", err)
-	}
-	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR()); err != nil {
-		return fmt.Errorf("failed to configure kube-proxy: %w", err)
-	}
-	if err := setup.KubeControllerManager(snap); err != nil {
-		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
-	}
-	if err := setup.KubeScheduler(snap); err != nil {
-		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
-	}
-	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil {
-		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
-	}
-	return nil
-}
-
 func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error {
 	// Start services
 	switch datastore {

src/k8s/pkg/k8sd/app/hooks_bootstrap.go

@@ -189,17 +189,20 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon
 	}
 
 	// Worker node services
-	if err := setup.Containerd(snap, nil); err != nil {
+	if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
 		return fmt.Errorf("failed to configure containerd: %w", err)
 	}
-	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil {
+	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil {
 		return fmt.Errorf("failed to configure kubelet: %w", err)
 	}
-	if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR); err != nil {
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR, joinConfig.ExtraNodeKubeProxyArgs); err != nil {
 		return fmt.Errorf("failed to configure kube-proxy: %w", err)
 	}
-	if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil {
-		return fmt.Errorf("failed to configure kube-proxy: %w", err)
+	if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil {
+		return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err)
+	}
+	if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil {
+		return fmt.Errorf("failed to write extra node config files: %w", err)
 	}
 
 	// TODO(berkayoz): remove the lock on cleanup
@@ -344,7 +347,7 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 	// Configure datastore
 	switch cfg.Datastore.GetType() {
 	case "k8s-dqlite":
-		if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil); err != nil {
+		if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil {
 			return fmt.Errorf("failed to configure k8s-dqlite: %w", err)
 		}
 	case "external":
@@ -353,8 +356,27 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 	}
 
 	// Configure services
-	if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil {
-		return fmt.Errorf("failed to configure services: %w", err)
+	if err := setup.Containerd(snap, nil, bootstrapConfig.ExtraNodeContainerdArgs); err != nil {
+		return fmt.Errorf("failed to configure containerd: %w", err)
+	}
+	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil {
+		return fmt.Errorf("failed to configure kubelet: %w", err)
+	}
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-proxy: %w", err)
+	}
+	if err := setup.KubeControllerManager(snap, bootstrapConfig.ExtraNodeKubeControllerManagerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
+	}
+	if err := setup.KubeScheduler(snap, bootstrapConfig.ExtraNodeKubeSchedulerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
+	}
+	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
+	}
+
+	if err := setup.ExtraNodeConfigFiles(snap, bootstrapConfig.ExtraNodeConfigFiles); err != nil {
+		return fmt.Errorf("failed to write extra node config files: %w", err)
 	}
 
 	// Write cluster configuration to dqlite

src/k8s/pkg/k8sd/app/hooks_join.go

@@ -134,7 +134,7 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
 		}
 
 		address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort())
-		if err := setup.K8sDqlite(snap, address, cluster); err != nil {
+		if err := setup.K8sDqlite(snap, address, cluster, joinConfig.ExtraNodeK8sDqliteArgs); err != nil {
 			return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err)
 		}
 	case "external":
@@ -143,8 +143,27 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
 	}
 
 	// Configure services
-	if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil {
-		return fmt.Errorf("failed to configure services: %w", err)
+	if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
+		return fmt.Errorf("failed to configure containerd: %w", err)
+	}
+	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil {
+		return fmt.Errorf("failed to configure kubelet: %w", err)
+	}
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), joinConfig.ExtraNodeKubeProxyArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-proxy: %w", err)
+	}
+	if err := setup.KubeControllerManager(snap, joinConfig.ExtraNodeKubeControllerManagerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
+	}
+	if err := setup.KubeScheduler(snap, joinConfig.ExtraNodeKubeSchedulerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
+	}
+	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil {
+		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
+	}
+
+	if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil {
+		return fmt.Errorf("failed to write extra node config files: %w", err)
 	}
 
 	if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil {

src/k8s/pkg/k8sd/setup/containerd.go

@@ -113,7 +113,7 @@ func containerdHostConfig(registry types.ContainerdRegistry) containerdHostsConf
 
 // Containerd configures configuration and arguments for containerd on the local node.
 // Optionally, a number of registry mirrors and auths can be configured.
-func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error {
+func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs map[string]*string) error {
 	configToml, err := os.OpenFile(path.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to open config.toml: %w", err)
@@ -138,6 +138,12 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error {
 		return fmt.Errorf("failed to write arguments file: %w", err)
 	}
 
+	// Apply extra arguments after the defaults, so they can override them.
+	updateArgs, deleteArgs := snaputil.ServiceArgsFromMap(extraArgs)
+	if _, err := snaputil.UpdateServiceArguments(snap, "containerd", updateArgs, deleteArgs); err != nil {
+		return fmt.Errorf("failed to write arguments file: %w", err)
+	}
+
 	cniBinary := path.Join(snap.CNIBinDir(), "cni")
 	if err := utils.CopyFile(snap.CNIPluginsBinary(), cniBinary); err != nil {
 		return fmt.Errorf("failed to copy cni plugin binary: %w", err)