Migrate config file to v3 schema

Ref: https://github.com/containerd/containerd/blob/release/2.0/docs/cri/config.md Signed-off-by: Brad Davidson <[email protected]>
k3s-io · Jan 17, 2025 · 0a0928b · 0a0928b
1 parent 8b63b77
commit 0a0928b
Show file tree

Hide file tree

Showing 7 changed files with 163 additions and 231 deletions.
diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go
@@ -27,17 +27,6 @@ const (
 	runtimesPath = "/usr/local/nvidia/toolkit:/opt/kwasm/bin"
 )
 
-func getContainerdArgs(cfg *config.Node) []string {
-	args := []string{
-		"containerd",
-		"-c", cfg.Containerd.Config,
-		"-a", cfg.Containerd.Address,
-		"--state", cfg.Containerd.State,
-		"--root", cfg.Containerd.Root,
-	}
-	return args
-}
-
 // SetupContainerdConfig generates the containerd.toml, using a template combined with various
 // runtime configurations and registry mirror settings provided by the administrator.
 func SetupContainerdConfig(cfg *config.Node) error {

diff --git a/pkg/agent/containerd/config_test.go b/pkg/agent/containerd/config_test.go
@@ -1473,12 +1473,18 @@ func Test_UnitGetHostConfigs(t *testing.T) {
 
 			nodeConfig := &config.Node{
 				Containerd: config.Containerd{
+					Address:  "/run/k3s/containerd/containerd.sock",
+					Opt:      "/var/lib/rancher/k3s/agent/containerd",
 					Registry: tempDir + "/hosts.d",
+					Root:     "/var/lib/rancher/k3s/agent/containerd",
+					State:    "/run/k3s/containerd",
 				},
 				AgentConfig: config.Agent{
 					ImageServiceSocket: "containerd-stargz-grpc.sock",
 					Registry:           registry.Registry,
 					Snapshotter:        "stargz",
+					CNIBinDir:          "/bin",
+					CNIConfDir:         "/var/lib/rancher/k3s/agent/etc/cni/net.d",
 				},
 			}
 

diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go
@@ -13,14 +13,6 @@ import (
 	"k8s.io/cri-client/pkg/util"
 )
 
-func getContainerdArgs(cfg *config.Node) []string {
-	args := []string{
-		"containerd",
-		"-c", cfg.Containerd.Config,
-	}
-	return args
-}
-
 // SetupContainerdConfig generates the containerd.toml, using a template combined with various
 // runtime configurations and registry mirror settings provided by the administrator.
 func SetupContainerdConfig(cfg *config.Node) error {

diff --git a/pkg/agent/containerd/containerd.go b/pkg/agent/containerd/containerd.go
@@ -50,7 +50,7 @@ const (
 // Run configures and starts containerd as a child process. Once it is up, images are preloaded
 // or pulled from files found in the agent images directory.
 func Run(ctx context.Context, cfg *config.Node) error {
-	args := getContainerdArgs(cfg)
+	args := []string{"containerd", "-c", cfg.Containerd.Config}
 	stdOut := io.Writer(os.Stdout)
 	stdErr := io.Writer(os.Stderr)
 

diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go
@@ -1,4 +1,5 @@
-//go:build linux
+//go:build linux && !windows_test
+// +build linux,!windows_test
 
 package templates
 
@@ -9,13 +10,21 @@ import (
 const ContainerdConfigTemplate = `
 {{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
-version = 2
+version = 3
+root = {{ printf "%q" .NodeConfig.Containerd.Root }}
+state = {{ printf "%q" .NodeConfig.Containerd.State }}
 
-[plugins."io.containerd.internal.v1.opt"]
+[grpc]
+  address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }}
+
+[plugins.'io.containerd.internal.v1.opt']
   path = "{{ .NodeConfig.Containerd.Opt }}"
-[plugins."io.containerd.grpc.v1.cri"]
+
+[plugins.'io.containerd.grpc.v1.cri']
   stream_server_address = "127.0.0.1"
   stream_server_port = "10010"
+
+[plugins.'io.containerd.cri.v1.runtime']
   enable_selinux = {{ .NodeConfig.SELinux }}
   enable_unprivileged_ports = {{ .EnableUnprivileged }}
   enable_unprivileged_icmp = {{ .EnableUnprivileged }}
@@ -24,85 +33,93 @@ version = 2
 {{- if .DisableCgroup}}
   disable_cgroup = true
 {{end}}
+
 {{- if .IsRunningInUserNS }}
   disable_apparmor = true
   restrict_oom_score_adj = true
 {{end}}
 
-{{- if .NodeConfig.AgentConfig.PauseImage }}
-  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-{{end}}
+{{ if .NodeConfig.AgentConfig.PauseImage }}
+[plugins.'io.containerd.cri.v1.images'.pinned_images]
+  sandbox = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{end -}}
 
-{{- if .NodeConfig.AgentConfig.Snapshotter }}
-[plugins."io.containerd.grpc.v1.cri".containerd]
+{{ if .NodeConfig.AgentConfig.Snapshotter }}
+[plugins.'io.containerd.cri.v1.images']
   snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
   disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}}
-  {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}}
-{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
-{{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
-[plugins."io.containerd.snapshotter.v1.stargz"]
-cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}"
-[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
-enable_keychain = true
-{{end}}
-
-[plugins."io.containerd.snapshotter.v1.stargz".registry]
-  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+{{end -}}
 
-{{ if .PrivateRegistryConfig }}
-{{range $k, $v := .PrivateRegistryConfig.Configs }}
-{{ if $v.Auth }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth]
-  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
-  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
-  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
-  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
-{{end}}
-{{end}}
-{{end}}
-{{end}}
-{{end}}
-
-{{- if not .NodeConfig.NoFlannel }}
-[plugins."io.containerd.grpc.v1.cri".cni]
+{{if not .NodeConfig.NoFlannel }}
+[plugins.'io.containerd.cri.v1.runtime'.cni]
   bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
   conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
 {{end}}
 
-{{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
-[plugins."io.containerd.service.v1.tasks-service"]
+{{ if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
+[plugins.'io.containerd.service.v1.tasks-service']
   {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}}
   {{ if .NodeConfig.Containerd.RDTConfig }}rdt_config_file = "{{ .NodeConfig.Containerd.RDTConfig }}"{{end}}
+{{end -}}
+
+{{ if .NodeConfig.DefaultRuntime }}
+[plugins.'io.containerd.cri.v1.runtime'.containerd]
+  default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"
 {{end}}
 
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc]
   runtime_type = "io.containerd.runc.v2"
 
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options]
   SystemdCgroup = {{ .SystemdCgroup }}
 
-[plugins."io.containerd.grpc.v1.cri".registry]
+{{- range $k, $v := .ExtraRuntimes }}
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{$k}}']
+  runtime_type = "{{$v.RuntimeType}}"
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{$k}}'.options]
+  BinaryName = "{{$v.BinaryName}}"
+  SystemdCgroup = {{ $.SystemdCgroup }}
+{{end}}
+
+[plugins.'io.containerd.grpc.v1.cri'.registry]
   config_path = "{{ .NodeConfig.Containerd.Registry }}"
 
-{{ if .PrivateRegistryConfig }}
-{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{- if .PrivateRegistryConfig }}
+{{- range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
-[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
+[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'{{$k}}'.auth]
   {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
   {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
-{{end}}
-{{end}}
-{{end}}
+{{end -}}
+{{end -}}
+{{end -}}
 
-{{range $k, $v := .ExtraRuntimes}}
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}"]
-  runtime_type = "{{$v.RuntimeType}}"
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}".options]
-  BinaryName = "{{$v.BinaryName}}"
-  SystemdCgroup = {{ $.SystemdCgroup }}
+{{- if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
+{{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
+[plugins.'io.containerd.snapshotter.v1.stargz']
+  cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}"
+
+[plugins.'io.containerd.snapshotter.v1.stargz'.cri_keychain]
+  enable_keychain = true
 {{end}}
+
+[plugins.'io.containerd.snapshotter.v1.stargz'.registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
+{{ if .PrivateRegistryConfig }}
+{{- range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins.'io.containerd.snapshotter.v1.stargz'.registry.configs.'{{$k}}'.auth]
+  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
+  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
+  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
+  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
+{{end -}}
+{{end -}}
+{{end -}}
+{{end -}}
 `
 
 // Linux config templates do not need fixups

diff --git a/pkg/agent/templates/templates_win.go b/pkg/agent/templates/templates_win.go
@@ -0,0 +1,86 @@
+//go:build windows || windows_test
+// +build windows windows_test
+
+// test with: go test -tags windows_test
+// note that this requires the filename not end with '_windows.go'
+
+package templates
+
+import (
+	"net/url"
+	"strings"
+	"text/template"
+)
+
+const ContainerdConfigTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
+version = 3
+root = {{ printf "%q" .NodeConfig.Containerd.Root }}
+state = {{ printf "%q" .NodeConfig.Containerd.State }}
+
+[grpc]
+  address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }}
+
+[plugins."io.containerd.internal.v1.opt"]
+  path = {{ printf "%q" .NodeConfig.Containerd.Opt }}
+
+[plugins.'io.containerd.grpc.v1.cri']
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+
+[plugins.'io.containerd.cri.v1.runtime']
+  enable_selinux = false
+
+[plugins.'io.containerd.grpc.v1.cri']
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+
+[plugins.'io.containerd.cri.v1.images']
+  snapshotter = "windows"
+
+[plugins.'io.containerd.cri.v1.runtime'.containerd]
+  default_runtime_name = "runhcs-wcow-process"
+
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runhcs-wcow-process]
+  runtime_type = "io.containerd.runhcs.v1"
+
+[plugins.'io.containerd.runtime.v2.task']
+  platforms = ["windows/amd64", "linux/amd64"]
+
+[plugins.'io.containerd.service.v1.diff-service']
+  default = ["windows", "windows-lcow"]
+
+[plugins.'io.containerd.cri.v1.runtime'.cni]
+  bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }}
+  conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }}
+
+[plugins.'io.containerd.grpc.v1.cri'.registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
+{{- if .PrivateRegistryConfig }}
+{{- range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'{{$k}}'.auth]
+  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
+  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
+  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
+  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
+{{end -}}
+{{end -}}
+{{end -}}
+`
+
+// Windows config templates need named pipe addresses fixed up
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		if strings.HasPrefix(s, "npipe:") {
+			u, err := url.Parse(s)
+			if err != nil {
+				return ""
+			}
+			return u.Path
+		}
+		return s
+	},
+}