From 5f2fb97bfd38d3fa36c0937f3941d4088e963ed3 Mon Sep 17 00:00:00 2001 From: Jun Shun Zhang <42823312+junshun@users.noreply.github.com> Date: Fri, 21 Apr 2023 07:43:10 -0400 Subject: [PATCH] Template credential-provider-config api version (#922) * Have credential-provider-config api version be dynamically generated based on kubernetes version. Always update KubeletCredentialProviderConfig and binaries in cases of version updates * using t.Setenv instead of os.Setenv for testing --- .../templates/daemonset.yaml | 2 + .../pkg/configurator/linux/linux.go | 38 +++++++++---- .../pkg/configurator/linux/linux_test.go | 57 +++++++++++++++++++ .../templates/credential-provider-config.yaml | 4 +- .../linux/testdata/expected-config-alpha.yaml | 17 ++++++ .../linux/testdata/expected-config-beta.yaml | 17 ++++++ .../expected-config-multiple-patterns.yaml | 4 +- .../linux/testdata/expected-config.yaml | 4 +- 8 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-alpha.yaml create mode 100644 credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-beta.yaml diff --git a/credentialproviderpackage/charts/credential-provider-package/templates/daemonset.yaml b/credentialproviderpackage/charts/credential-provider-package/templates/daemonset.yaml index 1bee2f3f..92df1c17 100644 --- a/credentialproviderpackage/charts/credential-provider-package/templates/daemonset.yaml +++ b/credentialproviderpackage/charts/credential-provider-package/templates/daemonset.yaml @@ -55,6 +55,8 @@ spec: value: '{{ join "," (index .Values.credential 0).matchImages }}' - name: DEFAULT_CACHE_DURATION value: {{(index .Values.credential 0).defaultCacheDuration}} + - name: K8S_VERSION + value: "v{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}" volumes: # Currently only one secret (aws-secret) is supported at this time - name: aws-creds diff --git a/credentialproviderpackage/pkg/configurator/linux/linux.go b/credentialproviderpackage/pkg/configurator/linux/linux.go index 2b426ac6..72e934ff 100644 --- a/credentialproviderpackage/pkg/configurator/linux/linux.go +++ b/credentialproviderpackage/pkg/configurator/linux/linux.go @@ -10,6 +10,7 @@ import ( "syscall" ps "github.com/mitchellh/go-ps" + "golang.org/x/mod/semver" "github.com/aws/eks-anywhere-packages/credentialproviderpackage/pkg/configurator" "github.com/aws/eks-anywhere-packages/credentialproviderpackage/pkg/constants" @@ -180,10 +181,23 @@ func copyBinaries() (string, error) { } func (c *linuxOS) createConfig() (string, error) { + k8sVersion := os.Getenv("K8S_VERSION") + apiVersion := "v1" + if semver.Compare(k8sVersion, "v1.26") <= 0 { + apiVersion = "v1beta1" + } + if semver.Compare(k8sVersion, "v1.24") <= 0 { + apiVersion = "v1alpha1" + } + if k8sVersion == "" { + apiVersion = "v1" + } + values := map[string]interface{}{ "profile": c.profile, "config": basePath + credOutFile, "home": basePath, + "apiVersion": apiVersion, "imagePattern": c.config.ImagePatterns, "cacheDuration": c.config.DefaultCacheDuration, } @@ -207,20 +221,22 @@ func (c *linuxOS) updateKubeletArguments(line string) string { args += " --feature-gates=KubeletCredentialProviders=true" } + val, err := c.createConfig() + if err != nil { + log.ErrorLogger.Printf("Error creating configuration %v", err) + } + // We want to upgrade the eksa owned configuration/binaries everytime however, + // we don't want to update what configuration is being pointed to in cases of a custom config if !strings.Contains(line, "image-credential-provider-config") { - val, err := c.createConfig() - if err != nil { - log.ErrorLogger.Printf("Error creating configuration %v", err) - } args += val + } - val, err = copyBinaries() - if err != nil { - log.ErrorLogger.Printf("Error coping binaries %v\n", err) - } - if !strings.Contains(line, "image-credential-provider-bin-dir") { - args += val - } + val, err = copyBinaries() + if err != nil { + log.ErrorLogger.Printf("Error coping binaries %v\n", err) + } + if !strings.Contains(line, "image-credential-provider-bin-dir") { + args += val } return args } diff --git a/credentialproviderpackage/pkg/configurator/linux/linux_test.go b/credentialproviderpackage/pkg/configurator/linux/linux_test.go index f7f1e398..1d18e760 100644 --- a/credentialproviderpackage/pkg/configurator/linux/linux_test.go +++ b/credentialproviderpackage/pkg/configurator/linux/linux_test.go @@ -30,6 +30,7 @@ func Test_linuxOS_updateKubeletArguments(t *testing.T) { args args outputConfigPath string configWantPath string + k8sVersion string want string }{ { @@ -99,6 +100,60 @@ func Test_linuxOS_updateKubeletArguments(t *testing.T) { configWantPath: "", want: "", }, + { + name: "test alpha api", + fields: fields{ + profile: "eksa-packages", + extraArgsPath: dir, + basePath: dir, + config: constants.CredentialProviderConfigOptions{ + ImagePatterns: []string{constants.DefaultImagePattern}, + DefaultCacheDuration: constants.DefaultCacheDuration, + }, + }, + args: args{line: ""}, + outputConfigPath: dir + "/" + credProviderFile, + configWantPath: "testdata/expected-config-alpha.yaml", + k8sVersion: "v1.24", + want: fmt.Sprintf(" --feature-gates=KubeletCredentialProviders=true "+ + "--image-credential-provider-config=%s%s", dir, credProviderFile), + }, + { + name: "test beta api", + fields: fields{ + profile: "eksa-packages", + extraArgsPath: dir, + basePath: dir, + config: constants.CredentialProviderConfigOptions{ + ImagePatterns: []string{constants.DefaultImagePattern}, + DefaultCacheDuration: constants.DefaultCacheDuration, + }, + }, + args: args{line: ""}, + outputConfigPath: dir + "/" + credProviderFile, + configWantPath: "testdata/expected-config-beta.yaml", + k8sVersion: "v1.26", + want: fmt.Sprintf(" --feature-gates=KubeletCredentialProviders=true "+ + "--image-credential-provider-config=%s%s", dir, credProviderFile), + }, + { + name: "test v1 api", + fields: fields{ + profile: "eksa-packages", + extraArgsPath: dir, + basePath: dir, + config: constants.CredentialProviderConfigOptions{ + ImagePatterns: []string{constants.DefaultImagePattern}, + DefaultCacheDuration: constants.DefaultCacheDuration, + }, + }, + args: args{line: ""}, + outputConfigPath: dir + "/" + credProviderFile, + configWantPath: "testdata/expected-config.yaml", + k8sVersion: "v1.27", + want: fmt.Sprintf(" --feature-gates=KubeletCredentialProviders=true "+ + "--image-credential-provider-config=%s%s", dir, credProviderFile), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -108,6 +163,8 @@ func Test_linuxOS_updateKubeletArguments(t *testing.T) { basePath: tt.fields.basePath, config: tt.fields.config, } + t.Setenv("K8S_VERSION", tt.k8sVersion) + if got := c.updateKubeletArguments(tt.args.line); got != tt.want { t.Errorf("updateKubeletArguments() = %v, want %v", got, tt.want) } diff --git a/credentialproviderpackage/pkg/configurator/linux/templates/credential-provider-config.yaml b/credentialproviderpackage/pkg/configurator/linux/templates/credential-provider-config.yaml index 3dc29279..35642f62 100644 --- a/credentialproviderpackage/pkg/configurator/linux/templates/credential-provider-config.yaml +++ b/credentialproviderpackage/pkg/configurator/linux/templates/credential-provider-config.yaml @@ -1,11 +1,11 @@ -apiVersion: kubelet.config.k8s.io/v1alpha1 +apiVersion: kubelet.config.k8s.io/{{.apiVersion}} kind: CredentialProviderConfig providers: - name: ecr-credential-provider matchImages:{{range $val := .imagePattern}} - "{{$val}}"{{end}} defaultCacheDuration: "{{.cacheDuration}}" - apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + apiVersion: credentialprovider.kubelet.k8s.io/{{.apiVersion}} env: - name: AWS_PROFILE value: {{.profile}} diff --git a/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-alpha.yaml b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-alpha.yaml new file mode 100644 index 00000000..2c7ffcd1 --- /dev/null +++ b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-alpha.yaml @@ -0,0 +1,17 @@ +apiVersion: kubelet.config.k8s.io/v1alpha1 +kind: CredentialProviderConfig +providers: + - name: ecr-credential-provider + matchImages: + - "*.dkr.ecr.*.amazonaws.com" + defaultCacheDuration: "30m" + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + env: + - name: AWS_PROFILE + value: eksa-packages + - name: AWS_CONFIG_FILE + value: /eksa-packages/aws-creds + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/eksa-packages + - name: HOME + value: /eksa-packages/ diff --git a/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-beta.yaml b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-beta.yaml new file mode 100644 index 00000000..baac3f92 --- /dev/null +++ b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-beta.yaml @@ -0,0 +1,17 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: CredentialProviderConfig +providers: + - name: ecr-credential-provider + matchImages: + - "*.dkr.ecr.*.amazonaws.com" + defaultCacheDuration: "30m" + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 + env: + - name: AWS_PROFILE + value: eksa-packages + - name: AWS_CONFIG_FILE + value: /eksa-packages/aws-creds + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/eksa-packages + - name: HOME + value: /eksa-packages/ diff --git a/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-multiple-patterns.yaml b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-multiple-patterns.yaml index 41f9a587..09a3cd7e 100644 --- a/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-multiple-patterns.yaml +++ b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config-multiple-patterns.yaml @@ -1,4 +1,4 @@ -apiVersion: kubelet.config.k8s.io/v1alpha1 +apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - name: ecr-credential-provider @@ -6,7 +6,7 @@ providers: - "1234567.dkr.ecr.us-east-1.amazonaws.com" - "7654321.dkr.ecr.us-west-2.amazonaws.com" defaultCacheDuration: "30m" - apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + apiVersion: credentialprovider.kubelet.k8s.io/v1 env: - name: AWS_PROFILE value: eksa-packages diff --git a/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config.yaml b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config.yaml index 2c7ffcd1..2f3d9295 100644 --- a/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config.yaml +++ b/credentialproviderpackage/pkg/configurator/linux/testdata/expected-config.yaml @@ -1,11 +1,11 @@ -apiVersion: kubelet.config.k8s.io/v1alpha1 +apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - name: ecr-credential-provider matchImages: - "*.dkr.ecr.*.amazonaws.com" defaultCacheDuration: "30m" - apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + apiVersion: credentialprovider.kubelet.k8s.io/v1 env: - name: AWS_PROFILE value: eksa-packages