diff --git a/src/k8s/api/v1/bootstrap_config.go b/src/k8s/api/v1/bootstrap_config.go index 190f7d4e76..8efa4797c6 100644 --- a/src/k8s/api/v1/bootstrap_config.go +++ b/src/k8s/api/v1/bootstrap_config.go @@ -55,6 +55,9 @@ type BootstrapConfig struct { KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d + ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"` + // Extra args to add to individual services (set any arg to null to delete) ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` diff --git a/src/k8s/api/v1/bootstrap_config_test.go b/src/k8s/api/v1/bootstrap_config_test.go index 6e46544047..400be832f1 100644 --- a/src/k8s/api/v1/bootstrap_config_test.go +++ b/src/k8s/api/v1/bootstrap_config_test.go @@ -48,6 +48,7 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) { K8sDqlitePort: utils.Pointer(9090), DatastoreType: utils.Pointer("k8s-dqlite"), ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeConfigFiles: map[string]string{"extra-node-config-file": "file-content"}, ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, diff --git a/src/k8s/api/v1/join_config.go b/src/k8s/api/v1/join_config.go index f59b03cb11..7a69aaa529 100644 --- a/src/k8s/api/v1/join_config.go +++ b/src/k8s/api/v1/join_config.go @@ -26,6 +26,9 @@ type ControlPlaneNodeJoinConfig struct { KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d + ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"` + // Extra args to add to individual services (set any arg to null to delete) ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"` ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"` @@ -44,6 +47,9 @@ type WorkerNodeJoinConfig struct { KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` + // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d + ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"` + // Extra args to add to individual services (set any arg to null to delete) ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"` ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"` diff --git a/src/k8s/cmd/k8s/k8s_bootstrap_test.go b/src/k8s/cmd/k8s/k8s_bootstrap_test.go index de5eda72ca..85fd178334 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap_test.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap_test.go @@ -72,6 +72,7 @@ var testCases = []testCase{ K8sDqlitePort: utils.Pointer(9090), DatastoreType: utils.Pointer("k8s-dqlite"), ExtraSANs: []string{"custom.kubernetes"}, + ExtraNodeConfigFiles: map[string]string{"extra-node-config-file": "/path/to/extra/config.yaml"}, ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")}, ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")}, ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")}, diff --git a/src/k8s/cmd/k8s/k8s_join_cluster.go b/src/k8s/cmd/k8s/k8s_join_cluster.go index 0bf5e4ac8d..9cbc7807d1 100644 --- a/src/k8s/cmd/k8s/k8s_join_cluster.go +++ b/src/k8s/cmd/k8s/k8s_join_cluster.go @@ -3,7 +3,6 @@ package k8s import ( "context" "fmt" - "io" "os" "time" @@ -77,23 +76,11 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { var joinClusterConfig string if opts.configFile != "" { - var b []byte - var err error - - if opts.configFile == "-" { - b, err = io.ReadAll(os.Stdin) - if err != nil { - cmd.PrintErrf("Error: Failed to read join configuration from stdin. \n\nThe error was: %v\n", err) - env.Exit(1) - return - } - } else { - b, err = os.ReadFile(opts.configFile) - if err != nil { - cmd.PrintErrf("Error: Failed to read join configuration from %q.\n\nThe error was: %v\n", opts.configFile, err) - env.Exit(1) - return - } + b, err := os.ReadFile(opts.configFile) + if err != nil { + cmd.PrintErrf("Error: Failed to read config file %s.\n\n The error was %v\n", opts.configFile, err) + env.Exit(1) + return } joinClusterConfig = string(b) } diff --git a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml index 07068680c8..61993523dc 100644 --- a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml +++ b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml @@ -31,6 +31,8 @@ k8s-dqlite-port: 9090 datastore-type: k8s-dqlite extra-sans: - custom.kubernetes +extra-node-config-files: + extra-node-config-file: /path/to/extra/config.yaml extra-node-kube-apiserver-args: --extra-kube-apiserver-arg: extra-kube-apiserver-value extra-node-kube-controller-manager-args: diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index ab90344dd2..9599b87041 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -201,6 +201,9 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) } + if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil { + return fmt.Errorf("failed to write extra node config files: %w", err) + } // TODO(berkayoz): remove the lock on cleanup if err := snaputil.MarkAsWorkerNode(snap, true); err != nil { @@ -372,6 +375,10 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot return fmt.Errorf("failed to configure kube-apiserver: %w", err) } + if err := setup.ExtraNodeConfigFiles(snap, bootstrapConfig.ExtraNodeConfigFiles); err != nil { + return fmt.Errorf("failed to write extra node config files: %w", err) + } + // Write cluster configuration to dqlite if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { if _, err := database.SetClusterConfig(ctx, tx, cfg); err != nil { diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 38d56a30f3..35d4d8e9af 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -162,6 +162,10 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { return fmt.Errorf("failed to configure kube-apiserver: %w", err) } + if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil { + return fmt.Errorf("failed to write extra node config files: %w", err) + } + if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil { return fmt.Errorf("failed to set snapd configuration from k8sd: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files.go b/src/k8s/pkg/k8sd/setup/util_extra_files.go new file mode 100644 index 0000000000..5fb0386552 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/util_extra_files.go @@ -0,0 +1,45 @@ +package setup + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/canonical/k8s/pkg/snap" +) + +// ExtraNodeConfigFiles writes the file contents to the specified filenames in the snap.ExtraFilesDir directory. +// The files are created with 0400 permissions and owned by root. +// The filenames must not contain any slashes to prevent path traversal. +func ExtraNodeConfigFiles(snap snap.Snap, files map[string]string) error { + for filename, content := range files { + if strings.Contains(filename, "/") { + return fmt.Errorf("file name %q must not contain any slashes (possible path-traversal prevented)", filename) + } + + filePath := path.Join(snap.ServiceExtraConfigDir(), filename) + // Create or truncate the file + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filePath, err) + } + defer file.Close() + + // Write the content to the file + _, err = file.WriteString(content) + if err != nil { + return fmt.Errorf("failed to write to file %s: %w", filePath, err) + } + + // Set file owner to root + if err := os.Chown(filePath, snap.UID(), snap.GID()); err != nil { + return fmt.Errorf("failed to change owner of file %s: %w", filePath, err) + } + + if err := os.Chmod(filePath, 0400); err != nil { + return fmt.Errorf("failed to change mode of file %s: %w", filePath, err) + } + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files_test.go b/src/k8s/pkg/k8sd/setup/util_extra_files_test.go new file mode 100644 index 0000000000..ad4fa38bb4 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/util_extra_files_test.go @@ -0,0 +1,73 @@ +package setup + +import ( + "os" + "path/filepath" + "testing" + + "github.com/canonical/k8s/pkg/snap/mock" + "github.com/onsi/gomega" +) + +func TestExtraNodeConfigFiles(t *testing.T) { + tests := []struct { + name string + files map[string]string + expectErr bool + errMessage string + }{ + { + name: "ValidFiles", + files: map[string]string{ + "config1": "content1", + "config2": "content2", + }, + expectErr: false, + }, + { + name: "InvalidFilename", + files: map[string]string{ + "invalid/config": "content", + }, + expectErr: true, + errMessage: "file name \"invalid/config\" must not contain any slashes", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + tmpDir := t.TempDir() + snap := &mock.Snap{ + Mock: mock.Mock{ + ServiceExtraConfigDir: tmpDir, + UID: os.Getuid(), + GID: os.Getgid(), + }, + } + + err := ExtraNodeConfigFiles(snap, tt.files) + if tt.expectErr { + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(err.Error()).To(gomega.ContainSubstring(tt.errMessage)) + } else { + g.Expect(err).ToNot(gomega.HaveOccurred()) + + for filename, content := range tt.files { + filePath := filepath.Join(tmpDir, filename) + + // Verify the file exists + info, err := os.Stat(filePath) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(info.Mode().Perm()).To(gomega.Equal(os.FileMode(0400))) + + // Verify the file content + actualContent, err := os.ReadFile(filePath) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(string(actualContent)).To(gomega.Equal(content)) + } + } + }) + } +} diff --git a/tests/integration/templates/bootstrap-all.yaml b/tests/integration/templates/bootstrap-all.yaml deleted file mode 100644 index 68ad1fbb43..0000000000 --- a/tests/integration/templates/bootstrap-all.yaml +++ /dev/null @@ -1,15 +0,0 @@ -cluster-config: - network: - enabled: true - dns: - enabled: true - ingress: - enabled: true - load-balancer: - enabled: true - local-storage: - enabled: true - gateway: - enabled: true - metrics-server: - enabled: true diff --git a/tests/integration/templates/bootstrap-session.yaml b/tests/integration/templates/bootstrap-session.yaml new file mode 100644 index 0000000000..ca569664f5 --- /dev/null +++ b/tests/integration/templates/bootstrap-session.yaml @@ -0,0 +1,33 @@ +# Contains the bootstrap configuration for the session instance of the integration tests. +# The session instance persists over test runs and is used to speed-up the integration tests. +cluster-config: + network: + enabled: true + dns: + enabled: true + ingress: + enabled: true + load-balancer: + enabled: true + local-storage: + enabled: true + gateway: + enabled: true + metrics-server: + enabled: true +extra-node-config-files: + bootstrap-extra-file.yaml: extra-args-test-file-content +extra-node-kube-apiserver-args: + --request-timeout: 2m +extra-node-kube-controller-manager-args: + --leader-elect-retry-period: 3s +extra-node-kube-scheduler-args: + --authorization-webhook-cache-authorized-ttl: 11s +extra-node-kube-proxy-args: + --config-sync-period: 14m +extra-node-kubelet-args: + --authentication-token-webhook-cache-ttl: 3m +extra-node-containerd-args: + --log-level: debug +extra-node-k8s-dqlite-args: + --watch-storage-available-size-interval: 6s diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py index 49926c3cd0..309cf982ba 100644 --- a/tests/integration/tests/conftest.py +++ b/tests/integration/tests/conftest.py @@ -126,9 +126,10 @@ def session_instance( instance = h.new_instance() util.setup_k8s_snap(instance, snap_path) - bootstrap_config_path = "/home/ubuntu/bootstrap-all-features.yaml" + bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml" instance.send_file( - (config.MANIFESTS_DIR / "bootstrap-all.yaml").as_posix(), bootstrap_config_path + (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(), + bootstrap_config_path, ) instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path]) diff --git a/tests/integration/tests/test_smoke.py b/tests/integration/tests/test_smoke.py index 2e424904b4..9bc9b4c853 100644 --- a/tests/integration/tests/test_smoke.py +++ b/tests/integration/tests/test_smoke.py @@ -17,3 +17,25 @@ def test_smoke(session_instance: harness.Instance): config = result.stdout.decode() assert len(config) > 0 assert "server: https://192.168.210.41" in config + + # Verify extra node configs + content = session_instance.exec( + ["cat", "/var/snap/k8s/common/args/conf.d/bootstrap-extra-file.yaml"], + capture_output=True, + ) + assert content.stdout.decode() == "extra-args-test-file-content" + + # For each service, verify that the extra arg was written to the args file. + for service, value in { + "kube-apiserver": "--request-timeout=2m", + "kube-controller-manager": "--leader-elect-retry-period=3s", + "kube-scheduler": "--authorization-webhook-cache-authorized-ttl=11s", + "kube-proxy": "--config-sync-period=14m", + "kubelet": "--authentication-token-webhook-cache-ttl=3m", + "containerd": "--log-level=debug", + "k8s-dqlite": "--watch-storage-available-size-interval=6s", + }.items(): + args = session_instance.exec( + ["cat", f"/var/snap/k8s/common/args/{service}"], capture_output=True + ) + assert value in args.stdout.decode() diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index c9a62b452a..599390e20d 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -223,8 +223,8 @@ def get_join_token( # Join an existing cluster. -def join_cluster(instance: harness.Instance, join_token: str): - instance.exec(["k8s", "join-cluster", join_token]) +def join_cluster(instance: harness.Instance, join_token: str, *args: str): + instance.exec(["k8s", "join-cluster", join_token, *args]) def get_default_cidr(instance: harness.Instance, instance_default_ip: str):