From 9209b5228e62b3d2e56939e9dbbc1435bba6a946 Mon Sep 17 00:00:00 2001 From: Tanvir Tatla Date: Wed, 17 Jan 2024 13:09:14 -0800 Subject: [PATCH] add clustercreator and kubeconfig writer --- Makefile | 5 +- cmd/eksctl-anywhere/cmd/createcluster.go | 6 +- pkg/clusterapi/name.go | 5 + pkg/clusterapi/name_test.go | 5 + pkg/clustermanager/cluster_creator.go | 66 ++++++++++++ pkg/clustermanager/cluster_creator_test.go | 78 ++++++++++++++ .../mocks/client_and_networking.go | 39 ++++++- pkg/dependencies/factory.go | 41 +++++++ pkg/dependencies/factory_test.go | 65 ++++++++++++ pkg/dependencies/testdata/cluster_docker.yaml | 35 ++++++ pkg/kubeconfig/kubeconfig.go | 7 ++ pkg/kubeconfig/kubeconfig_writer.go | 100 ++++++++++++++++++ pkg/kubeconfig/kubeconfig_writer_test.go | 98 +++++++++++++++++ pkg/kubeconfig/mocks/writer.go | 50 +++++++++ pkg/providers/docker/docker.go | 46 +++++++- pkg/providers/docker/docker_test.go | 43 ++++++++ pkg/providers/docker/mocks/client.go | 40 ++++++- pkg/workflows/interfaces/interfaces.go | 1 + pkg/workflows/interfaces/mocks/clients.go | 15 +++ pkg/workflows/workload/create.go | 2 +- pkg/workflows/workload/create_test.go | 4 +- pkg/workflows/workload/createcluster.go | 2 +- 22 files changed, 741 insertions(+), 12 deletions(-) create mode 100644 pkg/clustermanager/cluster_creator.go create mode 100644 pkg/clustermanager/cluster_creator_test.go create mode 100644 pkg/dependencies/testdata/cluster_docker.yaml create mode 100644 pkg/kubeconfig/kubeconfig_writer.go create mode 100644 pkg/kubeconfig/kubeconfig_writer_test.go create mode 100644 pkg/kubeconfig/mocks/writer.go diff --git a/Makefile b/Makefile index df6fed1f3fdf..6fa119f655fa 100644 --- a/Makefile +++ b/Makefile @@ -528,7 +528,7 @@ mocks: ## Generate mocks ${MOCKGEN} -destination=controllers/mocks/snow_machineconfig_controller.go -package=mocks -source "controllers/snow_machineconfig_controller.go" ${MOCKGEN} -destination=pkg/providers/mocks/providers.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers" Provider,DatacenterConfig,MachineConfig ${MOCKGEN} -destination=pkg/executables/mocks/executables.go -package=mocks "github.com/aws/eks-anywhere/pkg/executables" Executable,DockerClient,DockerContainer - ${MOCKGEN} -destination=pkg/providers/docker/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers/docker" ProviderClient,ProviderKubectlClient + ${MOCKGEN} -destination=pkg/providers/docker/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers/docker" ProviderClient,ProviderKubectlClient,KubeconfigReader ${MOCKGEN} -destination=pkg/providers/tinkerbell/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers/tinkerbell" ProviderKubectlClient,SSHAuthKeyGenerator ${MOCKGEN} -destination=pkg/providers/cloudstack/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers/cloudstack" ProviderCmkClient,ProviderKubectlClient ${MOCKGEN} -destination=pkg/providers/cloudstack/validator_mocks.go -package=cloudstack "github.com/aws/eks-anywhere/pkg/providers/cloudstack" ProviderValidator,ValidatorRegistry @@ -536,7 +536,7 @@ mocks: ## Generate mocks ${MOCKGEN} -destination=pkg/providers/vsphere/setupuser/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/providers/vsphere/setupuser" GovcClient ${MOCKGEN} -destination=pkg/govmomi/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/govmomi" VSphereClient,VMOMIAuthorizationManager,VMOMIFinder,VMOMISessionBuilder,VMOMIFinderBuilder,VMOMIAuthorizationManagerBuilder ${MOCKGEN} -destination=pkg/filewriter/mocks/filewriter.go -package=mocks "github.com/aws/eks-anywhere/pkg/filewriter" FileWriter - ${MOCKGEN} -destination=pkg/clustermanager/mocks/client_and_networking.go -package=mocks "github.com/aws/eks-anywhere/pkg/clustermanager" ClusterClient,Networking,AwsIamAuth,EKSAComponents,KubernetesClient,ClientFactory + ${MOCKGEN} -destination=pkg/clustermanager/mocks/client_and_networking.go -package=mocks "github.com/aws/eks-anywhere/pkg/clustermanager" ClusterClient,Networking,AwsIamAuth,EKSAComponents,KubernetesClient,ClientFactory,ClusterApplier ${MOCKGEN} -destination=pkg/gitops/flux/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/gitops/flux" FluxClient,KubeClient,GitOpsFluxClient,GitClient,Templater ${MOCKGEN} -destination=pkg/task/mocks/task.go -package=mocks "github.com/aws/eks-anywhere/pkg/task" Task ${MOCKGEN} -destination=pkg/bootstrapper/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/bootstrapper" KindClient,KubernetesClient @@ -613,6 +613,7 @@ mocks: ## Generate mocks ${MOCKGEN} -destination=pkg/registry/mocks/storage.go -package=mocks -source "pkg/registry/storage.go" StorageClient ${MOCKGEN} -destination=pkg/registry/mocks/repository.go -package=mocks oras.land/oras-go/v2/registry Repository ${MOCKGEN} -destination=controllers/mocks/nodeupgrade_controller.go -package=mocks -source "controllers/nodeupgrade_controller.go" RemoteClientRegistry + ${MOCKGEN} -destination=pkg/kubeconfig/mocks/writer.go -package=mocks -source "pkg/kubeconfig/kubeconfig.go" Writer .PHONY: verify-mocks verify-mocks: mocks ## Verify if mocks need to be updated diff --git a/cmd/eksctl-anywhere/cmd/createcluster.go b/cmd/eksctl-anywhere/cmd/createcluster.go index 5f8735e8944c..366fae9d83e1 100644 --- a/cmd/eksctl-anywhere/cmd/createcluster.go +++ b/cmd/eksctl-anywhere/cmd/createcluster.go @@ -188,7 +188,9 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er WithPackageInstaller(clusterSpec, cc.installPackages, cc.managementKubeconfig). WithValidatorClients(). WithCreateClusterDefaulter(createCLIConfig). - WithClusterApplier() + WithClusterApplier(). + WithKubeconfigWriter(clusterSpec.Cluster). + WithClusterCreator(clusterSpec.Cluster) if cc.timeoutOptions.noTimeouts { factory.WithNoTimeouts() @@ -263,9 +265,9 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er deps.ClusterManager, deps.GitOpsFlux, deps.Writer, - deps.ClusterApplier, deps.EksdInstaller, deps.PackageInstaller, + deps.ClusterCreator, ) err = createWorkloadCluster.Run(ctx, clusterSpec, createValidations) diff --git a/pkg/clusterapi/name.go b/pkg/clusterapi/name.go index 51b66cf0868f..99254f7adaf2 100644 --- a/pkg/clusterapi/name.go +++ b/pkg/clusterapi/name.go @@ -174,3 +174,8 @@ func EnsureNewNameIfChanged[M Object[M]](ctx context.Context, func ClusterCASecretName(clusterName string) string { return fmt.Sprintf("%s-ca", clusterName) } + +// ClusterKubeconfigSecretName returns the name of the kubeconfig secret for the cluster. +func ClusterKubeconfigSecretName(clusterName string) string { + return fmt.Sprintf("%s-kubeconfig", clusterName) +} diff --git a/pkg/clusterapi/name_test.go b/pkg/clusterapi/name_test.go index e0abc75b454a..80efa7e50f06 100644 --- a/pkg/clusterapi/name_test.go +++ b/pkg/clusterapi/name_test.go @@ -332,6 +332,11 @@ func TestClusterCASecretName(t *testing.T) { g.Expect(clusterapi.ClusterCASecretName("my-cluster")).To(Equal("my-cluster-ca")) } +func TestClusterKubeconfigSecretName(t *testing.T) { + g := NewWithT(t) + g.Expect(clusterapi.ClusterKubeconfigSecretName("my-cluster")).To(Equal("my-cluster-kubeconfig")) +} + func TestInitialTemplateNamesForWorkers(t *testing.T) { tests := []struct { name string diff --git a/pkg/clustermanager/cluster_creator.go b/pkg/clustermanager/cluster_creator.go new file mode 100644 index 000000000000..acd932b3e846 --- /dev/null +++ b/pkg/clustermanager/cluster_creator.go @@ -0,0 +1,66 @@ +package clustermanager + +import ( + "context" + + "github.com/aws/eks-anywhere/pkg/cluster" + "github.com/aws/eks-anywhere/pkg/filewriter" + "github.com/aws/eks-anywhere/pkg/kubeconfig" + "github.com/aws/eks-anywhere/pkg/types" +) + +// ClusterApplier is responsible for applying the cluster spec to the cluster. +type ClusterApplier interface { + Run(ctx context.Context, spec *cluster.Spec, managementCluster types.Cluster) error +} + +// ClusterCreator is responsible for applying the cluster config and writing the kubeconfig file. +type ClusterCreator struct { + ClusterApplier + kubeconfigWriter kubeconfig.Writer + fs filewriter.FileWriter +} + +// NewClusterCreator creates a ClusterCreator. +func NewClusterCreator(applier ClusterApplier, kubeconfigWriter kubeconfig.Writer, fs filewriter.FileWriter) *ClusterCreator { + return &ClusterCreator{ + ClusterApplier: applier, + kubeconfigWriter: kubeconfigWriter, + fs: fs, + } +} + +// CreateSync creates a workload cluster using the EKS-A controller and returns the types.Cluster object for that cluster. +func (cc ClusterCreator) CreateSync(ctx context.Context, spec *cluster.Spec, managementCluster *types.Cluster) (*types.Cluster, error) { + if err := cc.Run(ctx, spec, *managementCluster); err != nil { + return nil, err + } + + return cc.buildClusterAccess(ctx, spec.Cluster.Name, managementCluster) +} + +func (cc ClusterCreator) buildClusterAccess(ctx context.Context, clusterName string, management *types.Cluster) (*types.Cluster, error) { + cluster := &types.Cluster{ + Name: clusterName, + } + + fsOptions := []filewriter.FileOptionsFunc{filewriter.PersistentFile, filewriter.Permission0600} + fh, path, err := cc.fs.Create( + kubeconfig.FormatWorkloadClusterKubeconfigFilename(clusterName), + fsOptions..., + ) + if err != nil { + return nil, err + } + + defer fh.Close() + + err = cc.kubeconfigWriter.WriteKubeconfig(ctx, clusterName, management.KubeconfigFile, fh) + if err != nil { + return nil, err + } + + cluster.KubeconfigFile = path + + return cluster, nil +} diff --git a/pkg/clustermanager/cluster_creator_test.go b/pkg/clustermanager/cluster_creator_test.go new file mode 100644 index 000000000000..45c14b87bf51 --- /dev/null +++ b/pkg/clustermanager/cluster_creator_test.go @@ -0,0 +1,78 @@ +package clustermanager_test + +import ( + "context" + "fmt" + "io" + "os" + "testing" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + "k8s.io/utils/pointer" + + "github.com/aws/eks-anywhere/internal/test" + "github.com/aws/eks-anywhere/pkg/cluster" + "github.com/aws/eks-anywhere/pkg/clustermanager" + "github.com/aws/eks-anywhere/pkg/clustermanager/mocks" + "github.com/aws/eks-anywhere/pkg/filewriter" + mockswriter "github.com/aws/eks-anywhere/pkg/filewriter/mocks" + mockskubeconfig "github.com/aws/eks-anywhere/pkg/kubeconfig/mocks" + "github.com/aws/eks-anywhere/pkg/types" +) + +type clusterCreatorTest struct { + *WithT + ctx context.Context + spec *cluster.Spec + mgmtCluster *types.Cluster + applier *mocks.MockClusterApplier + writer *mockswriter.MockFileWriter + kubeconfigWriter *mockskubeconfig.MockWriter +} + +func newClusterCreator(t *testing.T, clusterName string) (*clustermanager.ClusterCreator, *clusterCreatorTest) { + ctrl := gomock.NewController(t) + cct := &clusterCreatorTest{ + WithT: NewWithT(t), + applier: mocks.NewMockClusterApplier(ctrl), + writer: mockswriter.NewMockFileWriter(ctrl), + kubeconfigWriter: mockskubeconfig.NewMockWriter(ctrl), + spec: test.NewClusterSpec(func(s *cluster.Spec) { + s.Cluster.Name = clusterName + }), + ctx: context.Background(), + mgmtCluster: &types.Cluster{ + KubeconfigFile: "my-config", + }, + } + + cc := clustermanager.NewClusterCreator(cct.applier, cct.kubeconfigWriter, cct.writer) + + return cc, cct +} + +func (cct *clusterCreatorTest) expectFileCreate(fileName, path string, w io.WriteCloser) { + cct.writer.EXPECT().Create(fileName, gomock.AssignableToTypeOf([]filewriter.FileOptionsFunc{})).Return(w, path, nil) +} + +func (cct *clusterCreatorTest) expectWriteKubeconfig(clusterName string, w io.Writer) { + cct.kubeconfigWriter.EXPECT().WriteKubeconfig(cct.ctx, clusterName, cct.mgmtCluster.KubeconfigFile, w).Return(nil) +} + +func (cct *clusterCreatorTest) expectApplierRun() { + cct.applier.EXPECT().Run(cct.ctx, cct.spec, *cct.mgmtCluster).Return(nil) +} + +func TestClusterCreatorCreateSync(t *testing.T) { + clusterName := "testCluster" + clusCreator, tt := newClusterCreator(t, clusterName) + path := "testpath" + writer := os.NewFile(uintptr(*pointer.Uint(0)), "test") + tt.expectApplierRun() + tt.expectWriteKubeconfig(clusterName, writer) + fileName := fmt.Sprintf("%s-eks-a-cluster.kubeconfig", clusterName) + tt.expectFileCreate(fileName, path, writer) + _, err := clusCreator.CreateSync(tt.ctx, tt.spec, tt.mgmtCluster) + tt.Expect(err).To(BeNil()) +} diff --git a/pkg/clustermanager/mocks/client_and_networking.go b/pkg/clustermanager/mocks/client_and_networking.go index ce856b73bf7a..d452b17512b0 100644 --- a/pkg/clustermanager/mocks/client_and_networking.go +++ b/pkg/clustermanager/mocks/client_and_networking.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/aws/eks-anywhere/pkg/clustermanager (interfaces: ClusterClient,Networking,AwsIamAuth,EKSAComponents,KubernetesClient,ClientFactory) +// Source: github.com/aws/eks-anywhere/pkg/clustermanager (interfaces: ClusterClient,Networking,AwsIamAuth,EKSAComponents,KubernetesClient,ClientFactory,ClusterApplier) // Package mocks is a generated GoMock package. package mocks @@ -1147,3 +1147,40 @@ func (mr *MockClientFactoryMockRecorder) BuildClientFromKubeconfig(arg0 interfac mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildClientFromKubeconfig", reflect.TypeOf((*MockClientFactory)(nil).BuildClientFromKubeconfig), arg0) } + +// MockClusterApplier is a mock of ClusterApplier interface. +type MockClusterApplier struct { + ctrl *gomock.Controller + recorder *MockClusterApplierMockRecorder +} + +// MockClusterApplierMockRecorder is the mock recorder for MockClusterApplier. +type MockClusterApplierMockRecorder struct { + mock *MockClusterApplier +} + +// NewMockClusterApplier creates a new mock instance. +func NewMockClusterApplier(ctrl *gomock.Controller) *MockClusterApplier { + mock := &MockClusterApplier{ctrl: ctrl} + mock.recorder = &MockClusterApplierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClusterApplier) EXPECT() *MockClusterApplierMockRecorder { + return m.recorder +} + +// Run mocks base method. +func (m *MockClusterApplier) Run(arg0 context.Context, arg1 *cluster.Spec, arg2 types.Cluster) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Run", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Run indicates an expected call of Run. +func (mr *MockClusterApplierMockRecorder) Run(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockClusterApplier)(nil).Run), arg0, arg1, arg2) +} diff --git a/pkg/dependencies/factory.go b/pkg/dependencies/factory.go index 1302fac4c7b9..11fc883c123b 100644 --- a/pkg/dependencies/factory.go +++ b/pkg/dependencies/factory.go @@ -112,6 +112,8 @@ type Dependencies struct { ExecutableBuilder *executables.ExecutablesBuilder CreateClusterDefaulter cli.CreateClusterDefaulter UpgradeClusterDefaulter cli.UpgradeClusterDefaulter + KubeconfigWriter kubeconfig.Writer + ClusterCreator *clustermanager.ClusterCreator } // KubeClients defines super struct that exposes all behavior. @@ -580,6 +582,45 @@ func (f *Factory) WithProvider(clusterConfigFile string, clusterConfig *v1alpha1 return f } +// WithKubeconfigWriter adds the KubeconfigReader dependency depending on the provider. +func (f *Factory) WithKubeconfigWriter(clusterConfig *v1alpha1.Cluster) *Factory { + f.WithUnAuthKubeClient() + if clusterConfig.Spec.DatacenterRef.Kind == v1alpha1.DockerDatacenterKind { + f.WithDocker() + } + + f.buildSteps = append(f.buildSteps, func(ctx context.Context) error { + if f.dependencies.KubeconfigWriter != nil { + return nil + } + writer := kubeconfig.NewClusterAPIKubeconfigSecretWriter(f.dependencies.UnAuthKubeClient) + switch clusterConfig.Spec.DatacenterRef.Kind { + case v1alpha1.DockerDatacenterKind: + f.dependencies.KubeconfigWriter = docker.NewKubeconfigWriter(f.dependencies.DockerClient, writer) + default: + f.dependencies.KubeconfigWriter = writer + } + return nil + }) + + return f +} + +// WithClusterCreator adds the ClusterCreator dependency. +func (f *Factory) WithClusterCreator(clusterConfig *v1alpha1.Cluster) *Factory { + f.WithClusterApplier().WithWriter().WithKubeconfigWriter(clusterConfig) + f.buildSteps = append(f.buildSteps, func(ctx context.Context) error { + if f.dependencies.ClusterCreator != nil { + return nil + } + + f.dependencies.ClusterCreator = clustermanager.NewClusterCreator(f.dependencies.ClusterApplier, f.dependencies.KubeconfigWriter, f.dependencies.Writer) + return nil + }) + + return f +} + func (f *Factory) WithDocker() *Factory { f.buildSteps = append(f.buildSteps, func(ctx context.Context) error { if f.dependencies.DockerClient != nil { diff --git a/pkg/dependencies/factory_test.go b/pkg/dependencies/factory_test.go index 0ffc35983c06..f2b992dc67dc 100644 --- a/pkg/dependencies/factory_test.go +++ b/pkg/dependencies/factory_test.go @@ -40,6 +40,7 @@ type factoryTest struct { type provider string const ( + docker provider = "docker" vsphere provider = "vsphere" tinkerbell provider = "tinkerbell" nutanix provider = "nutanix" @@ -60,6 +61,8 @@ func newTest(t *testing.T, p provider) *factoryTest { } switch p { + case docker: + f.clusterConfigFile = "testdata/cluster_docker.yaml" case vsphere: f.clusterConfigFile = "testdata/cluster_vsphere.yaml" case tinkerbell: @@ -86,6 +89,68 @@ func newTest(t *testing.T, p provider) *factoryTest { return f } +func TestFactoryBuildWithKubeconfigReader(t *testing.T) { + tests := []struct { + provider provider + }{ + { + provider: docker, + }, + { + provider: vsphere, + }, + } + + for _, tc := range tests { + tt := newTest(t, tc.provider) + deps, err := dependencies.NewFactory(). + WithLocalExecutables(). + WithKubeconfigWriter(tt.clusterSpec.Cluster). + Build(context.Background()) + tt.Expect(err).To(BeNil()) + tt.Expect(deps.KubeconfigWriter).NotTo(BeNil()) + } +} + +func TestFactoryBuildWithKubeconfigReaderAlreadyExists(t *testing.T) { + tt := newTest(t, docker) + factory := dependencies.NewFactory() + deps, err := factory.WithLocalExecutables().WithKubeconfigWriter(tt.clusterSpec.Cluster).Build(context.Background()) + tt.Expect(err).To(BeNil()) + tt.Expect(deps.KubeconfigWriter).ToNot(BeNil()) + deps, err = factory.WithKubeconfigWriter(tt.clusterSpec.Cluster).Build(context.Background()) + tt.Expect(err).To(BeNil()) + tt.Expect(deps.KubeconfigWriter).NotTo(BeNil()) +} + +func TestFactoryBuildWithClusterCreator(t *testing.T) { + tt := newTest(t, docker) + deps, err := dependencies.NewFactory(). + WithLocalExecutables(). + WithKubeconfigWriter(tt.clusterSpec.Cluster). + WithClusterCreator(tt.clusterSpec.Cluster). + Build(context.Background()) + tt.Expect(err).To(BeNil()) + tt.Expect(deps.ClusterCreator).NotTo(BeNil()) +} + +func TestFactoryBuildWithClusterCreatorAlreadyExists(t *testing.T) { + tt := newTest(t, docker) + factory := dependencies.NewFactory() + deps, _ := factory. + WithLocalExecutables(). + WithKubeconfigWriter(tt.clusterSpec.Cluster). + WithClusterCreator(tt.clusterSpec.Cluster). + Build(context.Background()) + tt.Expect(deps.ClusterCreator).NotTo(BeNil()) + deps, err := factory. + WithKubeconfigWriter(tt.clusterSpec.Cluster). + WithClusterCreator(tt.clusterSpec.Cluster). + Build(context.Background()) + tt.Expect(err).To(BeNil()) + tt.Expect(deps.ClusterCreator).NotTo(BeNil()) +} + func TestFactoryBuildWithProvidervSphere(t *testing.T) { tt := newTest(t, vsphere) deps, err := dependencies.NewFactory(). diff --git a/pkg/dependencies/testdata/cluster_docker.yaml b/pkg/dependencies/testdata/cluster_docker.yaml new file mode 100644 index 000000000000..c5ffcc9925de --- /dev/null +++ b/pkg/dependencies/testdata/cluster_docker.yaml @@ -0,0 +1,35 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: test-cluster + namespace: default +spec: + clusterNetwork: + cniConfig: + cilium: {} + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneConfiguration: + count: 1 + datacenterRef: + kind: DockerDatacenterConfig + name: test-cluster + kubernetesVersion: "1.21" + managementCluster: + name: test-cluster + workerNodeGroupConfigurations: + - count: 1 + name: md-0 + +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: DockerDatacenterConfig +metadata: + name: test-cluster + namespace: default +spec: {} + diff --git a/pkg/kubeconfig/kubeconfig.go b/pkg/kubeconfig/kubeconfig.go index 5cce623b713e..0f9330b26915 100644 --- a/pkg/kubeconfig/kubeconfig.go +++ b/pkg/kubeconfig/kubeconfig.go @@ -1,7 +1,9 @@ package kubeconfig import ( + "context" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -12,6 +14,11 @@ import ( "github.com/aws/eks-anywhere/pkg/validations" ) +// Writer reads the kubeconfig secret on a cluster and copies the contents to a writer. +type Writer interface { + WriteKubeconfig(ctx context.Context, clusterName, kubeconfig string, w io.Writer) error +} + // FromClusterFormat defines the format of the kubeconfig of the. const FromClusterFormat = "%s-eks-a-cluster.kubeconfig" diff --git a/pkg/kubeconfig/kubeconfig_writer.go b/pkg/kubeconfig/kubeconfig_writer.go new file mode 100644 index 000000000000..288044c67583 --- /dev/null +++ b/pkg/kubeconfig/kubeconfig_writer.go @@ -0,0 +1,100 @@ +package kubeconfig + +import ( + "bytes" + "context" + "io" + "time" + + corev1 "k8s.io/api/core/v1" + + "github.com/aws/eks-anywhere/pkg/clients/kubernetes" + "github.com/aws/eks-anywhere/pkg/clusterapi" + "github.com/aws/eks-anywhere/pkg/constants" + "github.com/aws/eks-anywhere/pkg/retrier" +) + +// ClientFactory builds Kubernetes clients. +type ClientFactory interface { + // BuildClientFromKubeconfig builds a Kubernetes client from a kubeconfig file. + BuildClientFromKubeconfig(kubeconfigPath string) (kubernetes.Client, error) +} + +// ClusterAPIKubeconfigSecretWriter reads the kubeconfig secret on a cluster and copies the contents to a writer. +type ClusterAPIKubeconfigSecretWriter struct { + client ClientFactory + timeout time.Duration + backoff time.Duration +} + +// WriterOpt allows to configure [KubeconfigWriter]. +type WriterOpt func(*ClusterAPIKubeconfigSecretWriter) + +// WithTimeout sets the optional timeout for a KubeconfigWriter. +func WithTimeout(timeout time.Duration) WriterOpt { + return func(writer *ClusterAPIKubeconfigSecretWriter) { + writer.timeout = timeout + } +} + +// WithBackoff sets the optional backoff duration for a KubeconfigWriter. +func WithBackoff(backoff time.Duration) WriterOpt { + return func(writer *ClusterAPIKubeconfigSecretWriter) { + writer.backoff = backoff + } +} + +// NewClusterAPIKubeconfigSecretWriter creates a ClusterAPIKubeconfigSecretWriter. +func NewClusterAPIKubeconfigSecretWriter(unauthClient ClientFactory, opts ...WriterOpt) ClusterAPIKubeconfigSecretWriter { + kr := &ClusterAPIKubeconfigSecretWriter{ + client: unauthClient, + timeout: time.Minute, + backoff: time.Second, + } + + for _, o := range opts { + o(kr) + } + + return *kr +} + +// WriteKubeconfig retrieves the contents of the specified cluster's kubeconfig from a secret and copies it to an io.Writer. +func (kr ClusterAPIKubeconfigSecretWriter) WriteKubeconfig(ctx context.Context, clusterName, kubeconfigPath string, w io.Writer) error { + rawKubeconfig, err := kr.GetClusterKubeconfig(ctx, clusterName, kubeconfigPath) + if err != nil { + return err + } + + if _, err := io.Copy(w, bytes.NewReader(rawKubeconfig)); err != nil { + return err + } + + return nil +} + +// GetClusterKubeconfig gets the cluster's kubeconfig from the secret. +func (kr ClusterAPIKubeconfigSecretWriter) GetClusterKubeconfig(ctx context.Context, clusterName, kubeconfigPath string) ([]byte, error) { + kubeconfigSecret := &corev1.Secret{} + var kubeClient kubernetes.Client + kubeClient, err := kr.client.BuildClientFromKubeconfig(kubeconfigPath) + if err != nil { + return nil, err + } + + err = retrier.New( + kr.timeout, + retrier.WithRetryPolicy(retrier.BackOffPolicy(kr.backoff)), + ).Retry(func() error { + if err = kubeClient.Get(ctx, clusterapi.ClusterKubeconfigSecretName(clusterName), constants.EksaSystemNamespace, kubeconfigSecret); err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + return kubeconfigSecret.Data["value"], nil +} diff --git a/pkg/kubeconfig/kubeconfig_writer_test.go b/pkg/kubeconfig/kubeconfig_writer_test.go new file mode 100644 index 000000000000..8c7f727a3328 --- /dev/null +++ b/pkg/kubeconfig/kubeconfig_writer_test.go @@ -0,0 +1,98 @@ +package kubeconfig_test + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/eks-anywhere/internal/test" + "github.com/aws/eks-anywhere/pkg/clients/kubernetes" + "github.com/aws/eks-anywhere/pkg/clusterapi" + "github.com/aws/eks-anywhere/pkg/clustermanager/mocks" + "github.com/aws/eks-anywhere/pkg/constants" + "github.com/aws/eks-anywhere/pkg/controller/clientutil" + "github.com/aws/eks-anywhere/pkg/kubeconfig" + "github.com/aws/eks-anywhere/pkg/types" +) + +type KubeconfigWriterTest struct { + *WithT + ctx context.Context + client kubernetes.Client + clientFactory *mocks.MockClientFactory + mgmtCluster *types.Cluster +} + +func newKubeconfigWriter(t *testing.T) (kubeconfig.Writer, *KubeconfigWriterTest) { + ctrl := gomock.NewController(t) + tt := &KubeconfigWriterTest{ + WithT: NewWithT(t), + ctx: context.Background(), + clientFactory: mocks.NewMockClientFactory(ctrl), + mgmtCluster: &types.Cluster{ + KubeconfigFile: "my-config", + }, + } + + writer := kubeconfig.NewClusterAPIKubeconfigSecretWriter(tt.clientFactory, kubeconfig.WithTimeout(time.Microsecond), kubeconfig.WithBackoff(time.Microsecond)) + + return writer, tt +} + +func (k *KubeconfigWriterTest) buildClient(err error, objs ...kubernetes.Object) { + k.client = test.NewFakeKubeClient(clientutil.ObjectsToClientObjects(objs)...) + k.clientFactory.EXPECT().BuildClientFromKubeconfig(k.mgmtCluster.KubeconfigFile).Return(k.client, err) +} + +func TestWriteKubeconfig(t *testing.T) { + tests := []struct { + name string + clusterName string + buildErr error + secret *corev1.Secret + expectErr bool + }{ + { + name: "success", + buildErr: nil, + clusterName: "test", + secret: &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: clusterapi.ClusterKubeconfigSecretName("test"), + Namespace: constants.EksaSystemNamespace, + }, + }, + }, + { + name: "build client fail", + buildErr: fmt.Errorf("failed to build client"), + secret: &corev1.Secret{}, + expectErr: true, + }, + { + name: "secret not found", + buildErr: nil, + clusterName: "test", + secret: &corev1.Secret{}, + expectErr: true, + }, + } + for _, tc := range tests { + writer, tt := newKubeconfigWriter(t) + buf := bytes.NewBuffer(make([]byte, tc.secret.Size())) + tt.buildClient(tc.buildErr, tc.secret) + err := writer.WriteKubeconfig(tt.ctx, tc.clusterName, tt.mgmtCluster.KubeconfigFile, buf) + if !tc.expectErr { + tt.Expect(err).To(BeNil()) + } else { + tt.Expect(err).ToNot(BeNil()) + } + } +} diff --git a/pkg/kubeconfig/mocks/writer.go b/pkg/kubeconfig/mocks/writer.go new file mode 100644 index 000000000000..3b28dd8b1d60 --- /dev/null +++ b/pkg/kubeconfig/mocks/writer.go @@ -0,0 +1,50 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/kubeconfig/kubeconfig.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + io "io" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockWriter is a mock of Writer interface. +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter. +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance. +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// WriteKubeconfig mocks base method. +func (m *MockWriter) WriteKubeconfig(ctx context.Context, clusterName, kubeconfig string, w io.Writer) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteKubeconfig", ctx, clusterName, kubeconfig, w) + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteKubeconfig indicates an expected call of WriteKubeconfig. +func (mr *MockWriterMockRecorder) WriteKubeconfig(ctx, clusterName, kubeconfig, w interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteKubeconfig", reflect.TypeOf((*MockWriter)(nil).WriteKubeconfig), ctx, clusterName, kubeconfig, w) +} diff --git a/pkg/providers/docker/docker.go b/pkg/providers/docker/docker.go index c4bf36d48bb9..d9e0db8b083b 100644 --- a/pkg/providers/docker/docker.go +++ b/pkg/providers/docker/docker.go @@ -1,9 +1,11 @@ package docker import ( + "bytes" "context" _ "embed" "fmt" + "io" "os" "regexp" @@ -52,6 +54,17 @@ type provider struct { templateBuilder *DockerTemplateBuilder } +// KubeconfigReader reads the kubeconfig secret from the cluster. +type KubeconfigReader interface { + GetClusterKubeconfig(ctx context.Context, clusterName, kubeconfigPath string) ([]byte, error) +} + +// KubeconfigWriter reads the kubeconfig secret on a docker cluster and copies the contents to a writer. +type KubeconfigWriter struct { + docker ProviderClient + reader KubeconfigReader +} + func (p *provider) InstallCustomProviderComponents(ctx context.Context, kubeconfigFile string) error { return nil } @@ -526,13 +539,42 @@ func (p *provider) UpdateKubeConfig(content *[]byte, clusterName string) error { if port, err := p.docker.GetDockerLBPort(ctx, clusterName); err != nil { return err } else { - getUpdatedKubeConfigContent(content, port) + updateKubeconfig(content, port) return nil } } +// NewKubeconfigWriter creates a KubeconfigWriter. +func NewKubeconfigWriter(docker ProviderClient, reader KubeconfigReader) KubeconfigWriter { + return KubeconfigWriter{ + reader: reader, + docker: docker, + } +} + +// WriteKubeconfig retrieves the contents of the specified cluster's kubeconfig from a secret and copies it to an io.Writer. +func (kr KubeconfigWriter) WriteKubeconfig(ctx context.Context, clusterName, kubeconfigPath string, w io.Writer) error { + rawkubeconfig, err := kr.reader.GetClusterKubeconfig(ctx, clusterName, kubeconfigPath) + if err != nil { + return err + } + + port, err := kr.docker.GetDockerLBPort(ctx, clusterName) + if err != nil { + return err + } + + updateKubeconfig(&rawkubeconfig, port) + + if _, err := io.Copy(w, bytes.NewReader(rawkubeconfig)); err != nil { + return err + } + + return nil +} + // this is required for docker provider. -func getUpdatedKubeConfigContent(content *[]byte, dockerLbPort string) { +func updateKubeconfig(content *[]byte, dockerLbPort string) { mc := regexp.MustCompile("server:.*") updatedConfig := mc.ReplaceAllString(string(*content), fmt.Sprintf("server: https://127.0.0.1:%s", dockerLbPort)) mc = regexp.MustCompile("certificate-authority-data:.*") diff --git a/pkg/providers/docker/docker_test.go b/pkg/providers/docker/docker_test.go index 22f7b01dc919..5953e91d5888 100644 --- a/pkg/providers/docker/docker_test.go +++ b/pkg/providers/docker/docker_test.go @@ -1,6 +1,7 @@ package docker_test import ( + "bytes" "context" _ "embed" "fmt" @@ -1011,3 +1012,45 @@ func TestTemplateBuilder_CertSANs(t *testing.T) { test.AssertContentToFile(t, string(data), tc.Output) } } + +func TestDockerWriteKubeconfig(t *testing.T) { + for _, tc := range []struct { + clusterName string + kubeconfigPath string + providerErr error + writerErr error + }{ + { + clusterName: "test", + kubeconfigPath: "test", + }, + { + clusterName: "test", + kubeconfigPath: "test", + writerErr: fmt.Errorf("failed to write kubeconfig"), + }, + { + clusterName: "test", + kubeconfigPath: "test", + providerErr: fmt.Errorf("failed to get LB port"), + }, + } { + g := NewWithT(t) + ctx := context.Background() + buf := bytes.NewBuffer([]byte{}) + mockCtrl := gomock.NewController(t) + dockerClient := dockerMocks.NewMockProviderClient(mockCtrl) + mocksWriter := dockerMocks.NewMockKubeconfigReader(mockCtrl) + mocksWriter.EXPECT().GetClusterKubeconfig(ctx, tc.clusterName, tc.kubeconfigPath).Return([]byte{}, tc.writerErr) + if tc.writerErr == nil { + dockerClient.EXPECT().GetDockerLBPort(ctx, tc.clusterName).Return("", tc.providerErr) + } + writer := docker.NewKubeconfigWriter(dockerClient, mocksWriter) + err := writer.WriteKubeconfig(ctx, tc.clusterName, tc.kubeconfigPath, buf) + if tc.writerErr == nil && tc.providerErr == nil { + g.Expect(err).ToNot(HaveOccurred()) + } else { + g.Expect(err).To(HaveOccurred()) + } + } +} diff --git a/pkg/providers/docker/mocks/client.go b/pkg/providers/docker/mocks/client.go index 1c848ce13c90..96d6d8b278dd 100644 --- a/pkg/providers/docker/mocks/client.go +++ b/pkg/providers/docker/mocks/client.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/aws/eks-anywhere/pkg/providers/docker (interfaces: ProviderClient,ProviderKubectlClient) +// Source: github.com/aws/eks-anywhere/pkg/providers/docker (interfaces: ProviderClient,ProviderKubectlClient,KubeconfigReader) // Package mocks is a generated GoMock package. package mocks @@ -171,3 +171,41 @@ func (mr *MockProviderKubectlClientMockRecorder) UpdateAnnotation(arg0, arg1, ar varargs := append([]interface{}{arg0, arg1, arg2, arg3}, arg4...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateAnnotation", reflect.TypeOf((*MockProviderKubectlClient)(nil).UpdateAnnotation), varargs...) } + +// MockKubeconfigReader is a mock of KubeconfigReader interface. +type MockKubeconfigReader struct { + ctrl *gomock.Controller + recorder *MockKubeconfigReaderMockRecorder +} + +// MockKubeconfigReaderMockRecorder is the mock recorder for MockKubeconfigReader. +type MockKubeconfigReaderMockRecorder struct { + mock *MockKubeconfigReader +} + +// NewMockKubeconfigReader creates a new mock instance. +func NewMockKubeconfigReader(ctrl *gomock.Controller) *MockKubeconfigReader { + mock := &MockKubeconfigReader{ctrl: ctrl} + mock.recorder = &MockKubeconfigReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockKubeconfigReader) EXPECT() *MockKubeconfigReaderMockRecorder { + return m.recorder +} + +// GetClusterKubeconfig mocks base method. +func (m *MockKubeconfigReader) GetClusterKubeconfig(arg0 context.Context, arg1, arg2 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClusterKubeconfig", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetClusterKubeconfig indicates an expected call of GetClusterKubeconfig. +func (mr *MockKubeconfigReaderMockRecorder) GetClusterKubeconfig(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterKubeconfig", reflect.TypeOf((*MockKubeconfigReader)(nil).GetClusterKubeconfig), arg0, arg1, arg2) +} diff --git a/pkg/workflows/interfaces/interfaces.go b/pkg/workflows/interfaces/interfaces.go index 53bf52cb2eea..74752f76253d 100644 --- a/pkg/workflows/interfaces/interfaces.go +++ b/pkg/workflows/interfaces/interfaces.go @@ -90,4 +90,5 @@ type ClusterUpgrader interface { // ClusterCreator creates the cluster and waits until it's ready. type ClusterCreator interface { Run(ctx context.Context, spec *cluster.Spec, managementCluster types.Cluster) error + CreateSync(ctx context.Context, spec *cluster.Spec, managementCluster *types.Cluster) (*types.Cluster, error) } diff --git a/pkg/workflows/interfaces/mocks/clients.go b/pkg/workflows/interfaces/mocks/clients.go index 4e09d90db032..2f9f7a179e8c 100644 --- a/pkg/workflows/interfaces/mocks/clients.go +++ b/pkg/workflows/interfaces/mocks/clients.go @@ -936,6 +936,21 @@ func (m *MockClusterCreator) EXPECT() *MockClusterCreatorMockRecorder { return m.recorder } +// CreateSync mocks base method. +func (m *MockClusterCreator) CreateSync(arg0 context.Context, arg1 *cluster.Spec, arg2 *types.Cluster) (*types.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSync", arg0, arg1, arg2) + ret0, _ := ret[0].(*types.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateSync indicates an expected call of CreateSync. +func (mr *MockClusterCreatorMockRecorder) CreateSync(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSync", reflect.TypeOf((*MockClusterCreator)(nil).CreateSync), arg0, arg1, arg2) +} + // Run mocks base method. func (m *MockClusterCreator) Run(arg0 context.Context, arg1 *cluster.Spec, arg2 types.Cluster) error { m.ctrl.T.Helper() diff --git a/pkg/workflows/workload/create.go b/pkg/workflows/workload/create.go index 0ac0d238e8a3..73f6c20e4fe5 100644 --- a/pkg/workflows/workload/create.go +++ b/pkg/workflows/workload/create.go @@ -25,9 +25,9 @@ type Create struct { func NewCreate(provider providers.Provider, clusterManager interfaces.ClusterManager, gitOpsManager interfaces.GitOpsManager, writer filewriter.FileWriter, - clusterCreator interfaces.ClusterCreator, eksdInstaller interfaces.EksdInstaller, packageInstaller interfaces.PackageInstaller, + clusterCreator interfaces.ClusterCreator, ) *Create { return &Create{ provider: provider, diff --git a/pkg/workflows/workload/create_test.go b/pkg/workflows/workload/create_test.go index 2dc4a8a6e108..8fd5666365e0 100644 --- a/pkg/workflows/workload/create_test.go +++ b/pkg/workflows/workload/create_test.go @@ -60,9 +60,9 @@ func newCreateTest(t *testing.T) *createTestSetup { clusterManager, gitOpsManager, writer, - clusterUpgrader, eksdInstaller, packageInstaller, + clusterUpgrader, ) for _, e := range featureEnvVars { @@ -99,7 +99,7 @@ func (c *createTestSetup) expectSetup() { } func (c *createTestSetup) expectCreateWorkloadCluster(err error) { - c.clusterCreator.EXPECT().Run(c.ctx, c.clusterSpec, *c.clusterSpec.ManagementCluster).Return(err) + c.clusterCreator.EXPECT().CreateSync(c.ctx, c.clusterSpec, c.clusterSpec.ManagementCluster).Return(nil, err) } func (c *createTestSetup) expectWriteWorkloadClusterConfig(err error) { diff --git a/pkg/workflows/workload/createcluster.go b/pkg/workflows/workload/createcluster.go index 325c05ee00e7..3a9c85e72079 100644 --- a/pkg/workflows/workload/createcluster.go +++ b/pkg/workflows/workload/createcluster.go @@ -13,7 +13,7 @@ type createCluster struct{} // Run createCluster performs actions needed to create the management cluster. func (c *createCluster) Run(ctx context.Context, commandContext *task.CommandContext) task.Task { logger.Info("Creating workload cluster") - if err := commandContext.ClusterCreator.Run(ctx, commandContext.ClusterSpec, *commandContext.ManagementCluster); err != nil { + if _, err := commandContext.ClusterCreator.CreateSync(ctx, commandContext.ClusterSpec, commandContext.ManagementCluster); err != nil { commandContext.SetError(err) return &workflows.CollectMgmtClusterDiagnosticsTask{} }