Adding support for multiple registry mirrors in bottlerocket

bootstrap/kubeadm/api/v1beta1/kubeadm_types.go

@@ -278,6 +278,21 @@ type RegistryMirrorConfiguration struct {
 
 	// CACert defines the CA cert for the registry mirror
 	CACert string `json:"caCert,omitempty"`
+
+	// Mirrors defines a list of image registry mirrors.
+	// +k8s:conversion-gen=false
+	// +optional
+	Mirrors []Mirror `json:"mirrors,omitempty"`
+}
+
+// Mirror holds the settings for mirroring a registry.
+type Mirror struct {
+	// Registry defines the registry we are mirroring to the endpoint.
+	Registry string `json:"registry,omitempty"`
+
+	// Endpoints defines the registry mirror endpoints to use for pulling images.
+	// Currently we support only one private registry. Hence endpoints would have only one entry.
+	Endpoints []string `json:"endpoints,omitempty"`
 }
 
 // ControlPlaneComponent holds settings common to control plane component of the cluster.

bootstrap/kubeadm/internal/bottlerocket/bootstrap.go

@@ -74,8 +74,11 @@ no-proxy = [{{stringsJoin .NoProxyEndpoints "," }}]
 {{- end -}}
 `
 	registryMirrorTemplate = `{{ define "registryMirrorSettings" -}}
-[settings.container-registry.mirrors]
-"public.ecr.aws" = ["https://{{.RegistryMirrorEndpoint}}"]
+{{- range $orig, $mirror := .RegistryMirrorMap }}
+[[settings.container-registry.mirrors]]
+registry = "{{ $orig }}"
+endpoint = [{{stringsJoin $mirror "," }}]
+{{- end -}}
 {{- end -}}
 `
 	registryMirrorCACertTemplate = `{{ define "registryMirrorCACertSettings" -}}
@@ -88,16 +91,21 @@ trusted=true
 	// to "public.ecr.aws" rather than the mirror's endpoint
 	// TODO: Once the bottlerocket fixes are in we need to remove the "public.ecr.aws" creds
 	registryMirrorCredentialsTemplate = `{{define "registryMirrorCredentialsSettings" -}}
+{{- range $orig, $mirror := .RegistryMirrorMap }}
+{{- if (eq $orig "public.ecr.aws")}}
 [[settings.container-registry.credentials]]
-registry = "public.ecr.aws"
-username = "{{.RegistryMirrorUsername}}"
-password = "{{.RegistryMirrorPassword}}"
+registry = "{{ $orig }}"
+username = "{{$.RegistryMirrorUsername}}"
+password = "{{$.RegistryMirrorPassword}}"
+{{- end }}
+{{- end }}
 [[settings.container-registry.credentials]]
 registry = "{{.RegistryMirrorEndpoint}}"
 username = "{{.RegistryMirrorUsername}}"
 password = "{{.RegistryMirrorPassword}}"
 {{- end -}}
 `
+
 	nodeLabelsTemplate = `{{ define "nodeLabelSettings" -}}
 [settings.kubernetes.node-labels]
 {{.NodeLabels}}
@@ -153,7 +161,7 @@ trusted = true
 {{- end -}}
 
 
-{{- if (ne .RegistryMirrorEndpoint "")}}
+{{- if .RegistryMirrorMap}}
 {{template "registryMirrorSettings" .}}
 {{- end -}}
 

bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go

@@ -134,8 +134,10 @@ provider-id = "PROVIDERID"
 
 [settings.network]
 hostname = "hostname"
-[settings.container-registry.mirrors]
-"public.ecr.aws" = ["https://REGISTRY_ENDPOINT"]
+
+[[settings.container-registry.mirrors]]
+registry = "public.ecr.aws"
+endpoint = ["REGISTRY_ENDPOINT"]
 [settings.pki.registry-mirror-ca]
 data = "UkVHSVNUUllfQ0E="
 trusted=true`
@@ -162,11 +164,14 @@ provider-id = "PROVIDERID"
 
 [settings.network]
 hostname = "hostname"
-[settings.container-registry.mirrors]
-"public.ecr.aws" = ["https://REGISTRY_ENDPOINT"]
+
+[[settings.container-registry.mirrors]]
+registry = "public.ecr.aws"
+endpoint = ["REGISTRY_ENDPOINT"]
 [settings.pki.registry-mirror-ca]
 data = "UkVHSVNUUllfQ0E="
 trusted=true
+
 [[settings.container-registry.credentials]]
 registry = "public.ecr.aws"
 username = "admin"
@@ -340,6 +345,39 @@ trusted = true
 [settings.pki.bundle2]
 data = "MTIzNDU2"
 trusted = true`
+
+	registryMirrorMultipleMirrorsUserData = `
+[settings.host-containers.admin]
+enabled = true
+superpowered = true
+source = "ADMIN_REPO:ADMIN_TAG"
+user-data = "CnsKCSJzc2giOiB7CgkJImF1dGhvcml6ZWQta2V5cyI6IFsic3NoLXJzYSBBQUEuLi4iXQoJfQp9"
+[settings.host-containers.kubeadm-bootstrap]
+enabled = true
+superpowered = true
+source = "BOOTSTRAP_REPO:BOOTSTRAP_TAG"
+user-data = "Qk9UVExFUk9DS0VUX0JPT1RTVFJBUF9VU0VSREFUQQ=="
+
+[settings.kubernetes]
+cluster-domain = "cluster.local"
+standalone-mode = true
+authentication-mode = "tls"
+server-tls-bootstrap = false
+pod-infra-container-image = "PAUSE_REPO:PAUSE_TAG"
+provider-id = "PROVIDERID"
+
+[settings.network]
+hostname = "hostname"
+
+[[settings.container-registry.mirrors]]
+registry = "docker.io"
+endpoint = ["REGISTRY_ENDPOINT"]
+[[settings.container-registry.mirrors]]
+registry = "public.ecr.aws"
+endpoint = ["REGISTRY_ENDPOINT"]
+[settings.pki.registry-mirror-ca]
+data = "UkVHSVNUUllfQ0E="
+trusted=true`
 )
 
 var (
@@ -622,6 +660,32 @@ func TestGetBottlerocketNodeUserData(t *testing.T) {
 			},
 			output: userDataWithCertBundle,
 		},
+		{
+			name: "with multiple registries to mirror",
+			config: &BottlerocketConfig{
+				BottlerocketAdmin:     brAdmin,
+				BottlerocketBootstrap: brBootstrap,
+				Hostname:              hostname,
+				Pause:                 pause,
+				KubeletExtraArgs: map[string]string{
+					"provider-id": "PROVIDERID",
+				},
+				RegistryMirrorConfiguration: bootstrapv1.RegistryMirrorConfiguration{
+					CACert: "REGISTRY_CA",
+					Mirrors: []bootstrapv1.Mirror{
+						{
+							Registry:  "docker.io",
+							Endpoints: []string{"REGISTRY_ENDPOINT"},
+						},
+						{
+							Registry:  "public.ecr.aws",
+							Endpoints: []string{"REGISTRY_ENDPOINT"},
+						},
+					},
+				},
+			},
+			output: registryMirrorMultipleMirrorsUserData,
+		},
 	}
 	for _, testcase := range testcases {
 		t.Run(testcase.name, func(t *testing.T) {