Add extra config files parameters

src/k8s/api/v1/bootstrap_config.go

@@ -55,6 +55,9 @@ type BootstrapConfig struct {
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
 
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
 	// Extra args to add to individual services (set any arg to null to delete)
 	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
 	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`

src/k8s/api/v1/bootstrap_config_test.go

@@ -48,6 +48,7 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) {
 		K8sDqlitePort:                      utils.Pointer(9090),
 		DatastoreType:                      utils.Pointer("k8s-dqlite"),
 		ExtraSANs:                          []string{"custom.kubernetes"},
+		ExtraNodeConfigFiles:               map[string]string{"extra-node-config-file": "file-content"},
 		ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
 		ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
 		ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},

src/k8s/api/v1/join_config.go

@@ -26,6 +26,9 @@ type ControlPlaneNodeJoinConfig struct {
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
 
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
 	// Extra args to add to individual services (set any arg to null to delete)
 	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
 	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
@@ -44,6 +47,9 @@ type WorkerNodeJoinConfig struct {
 	KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
 	KubeProxyClientKey  *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
 
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
 	// Extra args to add to individual services (set any arg to null to delete)
 	ExtraNodeKubeProxyArgs         map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
 	ExtraNodeKubeletArgs           map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`

src/k8s/cmd/k8s/k8s_bootstrap_test.go

@@ -72,6 +72,7 @@ var testCases = []testCase{
 			K8sDqlitePort:                      utils.Pointer(9090),
 			DatastoreType:                      utils.Pointer("k8s-dqlite"),
 			ExtraSANs:                          []string{"custom.kubernetes"},
+			ExtraNodeConfigFiles:               map[string]string{"extra-node-config-file": "/path/to/extra/config.yaml"},
 			ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
 			ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
 			ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},

src/k8s/cmd/k8s/k8s_join_cluster.go

@@ -3,7 +3,6 @@ package k8s
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
 	"time"
 
@@ -77,23 +76,11 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 
 			var joinClusterConfig string
 			if opts.configFile != "" {
-				var b []byte
-				var err error
-
-				if opts.configFile == "-" {
-					b, err = io.ReadAll(os.Stdin)
-					if err != nil {
-						cmd.PrintErrf("Error: Failed to read join configuration from stdin. \n\nThe error was: %v\n", err)
-						env.Exit(1)
-						return
-					}
-				} else {
-					b, err = os.ReadFile(opts.configFile)
-					if err != nil {
-						cmd.PrintErrf("Error: Failed to read join configuration from %q.\n\nThe error was: %v\n", opts.configFile, err)
-						env.Exit(1)
-						return
-					}
+				b, err := os.ReadFile(opts.configFile)
+				if err != nil {
+					cmd.PrintErrf("Error: Failed to read config file %s.\n\n The error was %v\n", opts.configFile, err)
+					env.Exit(1)
+					return
 				}
 				joinClusterConfig = string(b)
 			}

src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml

@@ -31,6 +31,8 @@ k8s-dqlite-port: 9090
 datastore-type: k8s-dqlite
 extra-sans:
 - custom.kubernetes
+extra-node-config-files:
+  extra-node-config-file: /path/to/extra/config.yaml
 extra-node-kube-apiserver-args:
   --extra-kube-apiserver-arg: extra-kube-apiserver-value
 extra-node-kube-controller-manager-args:

src/k8s/pkg/k8sd/app/hooks_bootstrap.go

@@ -201,6 +201,9 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon
 	if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil {
 		return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err)
 	}
+	if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil {
+		return fmt.Errorf("failed to write extra node config files: %w", err)
+	}
 
 	// TODO(berkayoz): remove the lock on cleanup
 	if err := snaputil.MarkAsWorkerNode(snap, true); err != nil {
@@ -372,6 +375,10 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
 	}
 
+	if err := setup.ExtraNodeConfigFiles(snap, bootstrapConfig.ExtraNodeConfigFiles); err != nil {
+		return fmt.Errorf("failed to write extra node config files: %w", err)
+	}
+
 	// Write cluster configuration to dqlite
 	if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
 		if _, err := database.SetClusterConfig(ctx, tx, cfg); err != nil {

src/k8s/pkg/k8sd/app/hooks_join.go

@@ -162,6 +162,10 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
 		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
 	}
 
+	if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil {
+		return fmt.Errorf("failed to write extra node config files: %w", err)
+	}
+
 	if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil {
 		return fmt.Errorf("failed to set snapd configuration from k8sd: %w", err)
 	}

src/k8s/pkg/k8sd/setup/util_extra_files.go

@@ -0,0 +1,45 @@
+package setup
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/canonical/k8s/pkg/snap"
+)
+
+// ExtraNodeConfigFiles writes the file contents to the specified filenames in the snap.ExtraFilesDir directory.
+// The files are created with 0400 permissions and owned by root.
+// The filenames must not contain any slashes to prevent path traversal.
+func ExtraNodeConfigFiles(snap snap.Snap, files map[string]string) error {
+	for filename, content := range files {
+		if strings.Contains(filename, "/") {
+			return fmt.Errorf("file name %q must not contain any slashes (possible path-traversal prevented)", filename)
+		}
+
+		filePath := path.Join(snap.ServiceExtraConfigDir(), filename)
+		// Create or truncate the file
+		file, err := os.Create(filePath)
+		if err != nil {
+			return fmt.Errorf("failed to create file %s: %w", filePath, err)
+		}
+		defer file.Close()
+
+		// Write the content to the file
+		_, err = file.WriteString(content)
+		if err != nil {
+			return fmt.Errorf("failed to write to file %s: %w", filePath, err)
+		}
+
+		// Set file owner to root
+		if err := os.Chown(filePath, snap.UID(), snap.GID()); err != nil {
+			return fmt.Errorf("failed to change owner of file %s: %w", filePath, err)
+		}
+
+		if err := os.Chmod(filePath, 0400); err != nil {
+			return fmt.Errorf("failed to change mode of file %s: %w", filePath, err)
+		}
+	}
+	return nil
+}

src/k8s/pkg/k8sd/setup/util_extra_files_test.go

@@ -0,0 +1,73 @@
+package setup
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/canonical/k8s/pkg/snap/mock"
+	"github.com/onsi/gomega"
+)
+
+func TestExtraNodeConfigFiles(t *testing.T) {
+	tests := []struct {
+		name       string
+		files      map[string]string
+		expectErr  bool
+		errMessage string
+	}{
+		{
+			name: "ValidFiles",
+			files: map[string]string{
+				"config1": "content1",
+				"config2": "content2",
+			},
+			expectErr: false,
+		},
+		{
+			name: "InvalidFilename",
+			files: map[string]string{
+				"invalid/config": "content",
+			},
+			expectErr:  true,
+			errMessage: "file name \"invalid/config\" must not contain any slashes",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := gomega.NewGomegaWithT(t)
+
+			tmpDir := t.TempDir()
+			snap := &mock.Snap{
+				Mock: mock.Mock{
+					ServiceExtraConfigDir: tmpDir,
+					UID:                   os.Getuid(),
+					GID:                   os.Getgid(),
+				},
+			}
+
+			err := ExtraNodeConfigFiles(snap, tt.files)
+			if tt.expectErr {
+				g.Expect(err).To(gomega.HaveOccurred())
+				g.Expect(err.Error()).To(gomega.ContainSubstring(tt.errMessage))
+			} else {
+				g.Expect(err).ToNot(gomega.HaveOccurred())
+
+				for filename, content := range tt.files {
+					filePath := filepath.Join(tmpDir, filename)
+
+					// Verify the file exists
+					info, err := os.Stat(filePath)
+					g.Expect(err).ToNot(gomega.HaveOccurred())
+					g.Expect(info.Mode().Perm()).To(gomega.Equal(os.FileMode(0400)))
+
+					// Verify the file content
+					actualContent, err := os.ReadFile(filePath)
+					g.Expect(err).ToNot(gomega.HaveOccurred())
+					g.Expect(string(actualContent)).To(gomega.Equal(content))
+				}
+			}
+		})
+	}
+}

tests/integration/templates/bootstrap-session.yaml

@@ -0,0 +1,33 @@
+# Contains the bootstrap configuration for the session instance of the integration tests.
+# The session instance persists over test runs and is used to speed-up the integration tests.
+cluster-config:
+  network:
+    enabled: true
+  dns:
+    enabled: true
+  ingress:
+    enabled: true
+  load-balancer:
+    enabled: true
+  local-storage:
+    enabled: true
+  gateway:
+    enabled: true
+  metrics-server:
+    enabled: true
+extra-node-config-files:
+  bootstrap-extra-file.yaml: extra-args-test-file-content
+extra-node-kube-apiserver-args:
+  --request-timeout: 2m
+extra-node-kube-controller-manager-args:
+  --leader-elect-retry-period: 3s
+extra-node-kube-scheduler-args:
+  --authorization-webhook-cache-authorized-ttl: 11s
+extra-node-kube-proxy-args:
+  --config-sync-period: 14m
+extra-node-kubelet-args:
+  --authentication-token-webhook-cache-ttl: 3m
+extra-node-containerd-args:
+  --log-level: debug
+extra-node-k8s-dqlite-args:
+  --watch-storage-available-size-interval: 6s

tests/integration/tests/conftest.py

@@ -126,9 +126,10 @@ def session_instance(
     instance = h.new_instance()
     util.setup_k8s_snap(instance, snap_path)
 
-    bootstrap_config_path = "/home/ubuntu/bootstrap-all-features.yaml"
+    bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml"
     instance.send_file(
-        (config.MANIFESTS_DIR / "bootstrap-all.yaml").as_posix(), bootstrap_config_path
+        (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(),
+        bootstrap_config_path,
     )
 
     instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path])

tests/integration/tests/test_smoke.py

@@ -17,3 +17,25 @@ def test_smoke(session_instance: harness.Instance):
     config = result.stdout.decode()
     assert len(config) > 0
     assert "server: https://192.168.210.41" in config
+
+    # Verify extra node configs
+    content = session_instance.exec(
+        ["cat", "/var/snap/k8s/common/args/conf.d/bootstrap-extra-file.yaml"],
+        capture_output=True,
+    )
+    assert content.stdout.decode() == "extra-args-test-file-content"
+
+    # For each service, verify that the extra arg was written to the args file.
+    for service, value in {
+        "kube-apiserver": "--request-timeout=2m",
+        "kube-controller-manager": "--leader-elect-retry-period=3s",
+        "kube-scheduler": "--authorization-webhook-cache-authorized-ttl=11s",
+        "kube-proxy": "--config-sync-period=14m",
+        "kubelet": "--authentication-token-webhook-cache-ttl=3m",
+        "containerd": "--log-level=debug",
+        "k8s-dqlite": "--watch-storage-available-size-interval=6s",
+    }.items():
+        args = session_instance.exec(
+            ["cat", f"/var/snap/k8s/common/args/{service}"], capture_output=True
+        )
+        assert value in args.stdout.decode()

tests/integration/tests/test_util/util.py

@@ -223,8 +223,8 @@ def get_join_token(
 
 
 # Join an existing cluster.
-def join_cluster(instance: harness.Instance, join_token: str):
-    instance.exec(["k8s", "join-cluster", join_token])
+def join_cluster(instance: harness.Instance, join_token: str, *args: str):
+    instance.exec(["k8s", "join-cluster", join_token, *args])
 
 
 def get_default_cidr(instance: harness.Instance, instance_default_ip: str):