From b79b377876cbfa5c8a713588c55127603da906ea Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 18 Jan 2024 11:29:56 -0800 Subject: [PATCH] refactoring upgrade rollout strategy tests --- pkg/providers/docker/controlplane_test.go | 27 ++ pkg/providers/docker/docker_test.go | 32 -- .../valid_deployment_cp_rollout_expected.yaml | 316 ------------------ .../valid_deployment_md_rollout_expected.yaml | 61 ---- pkg/providers/docker/workers_test.go | 45 +++ 5 files changed, 72 insertions(+), 409 deletions(-) delete mode 100644 pkg/providers/docker/testdata/valid_deployment_cp_rollout_expected.yaml delete mode 100644 pkg/providers/docker/testdata/valid_deployment_md_rollout_expected.yaml diff --git a/pkg/providers/docker/controlplane_test.go b/pkg/providers/docker/controlplane_test.go index c93ba2f134bf5..e379cd09fc78e 100644 --- a/pkg/providers/docker/controlplane_test.go +++ b/pkg/providers/docker/controlplane_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" @@ -271,6 +272,32 @@ func TestControlPlaneSpecRegistryMirrorConfiguration(t *testing.T) { } } +func TestControlPlaneUpgradeRolloutStrategy(t *testing.T) { + g := NewWithT(t) + logger := test.NewNullLogger() + ctx := context.Background() + client := test.NewFakeKubeClient() + spec := testClusterSpec(func(s *cluster.Spec) { + s.Cluster.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy = &anywherev1.ControlPlaneUpgradeRolloutStrategy{ + RollingUpdate: anywherev1.ControlPlaneRollingUpdateParams{ + MaxSurge: 1, + }, + } + }) + + cp, err := docker.ControlPlaneSpec(ctx, logger, client, spec) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cp).NotTo(BeNil()) + g.Expect(cp.KubeadmControlPlane).To(Equal(kubeadmControlPlane(func(k *controlplanev1.KubeadmControlPlane) { + maxSurge := intstr.FromInt(1) + k.Spec.RolloutStrategy = &controlplanev1.RolloutStrategy{ + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &maxSurge, + }, + } + }))) +} + func testClusterSpec(opts ...test.ClusterSpecOpt) *cluster.Spec { name := "test" namespace := "test-namespace" diff --git a/pkg/providers/docker/docker_test.go b/pkg/providers/docker/docker_test.go index c3d5d648df328..22f7b01dc919c 100644 --- a/pkg/providers/docker/docker_test.go +++ b/pkg/providers/docker/docker_test.go @@ -327,38 +327,6 @@ func TestProviderGenerateDeploymentFileSuccessUpdateMachineTemplate(t *testing.T wantCPFile: "testdata/valid_deployment_cp_expected.yaml", wantMDFile: "testdata/valid_autoscaler_deployment_md_expected.yaml", }, - { - testName: "valid config with rollout", - clusterSpec: test.NewClusterSpec(func(s *cluster.Spec) { - s.Cluster.Name = "test-cluster" - s.Cluster.Spec.KubernetesVersion = "1.19" - s.Cluster.Spec.ClusterNetwork.Pods.CidrBlocks = []string{"192.168.0.0/16"} - s.Cluster.Spec.ClusterNetwork.Services.CidrBlocks = []string{"10.128.0.0/12"} - s.Cluster.Spec.ControlPlaneConfiguration.Count = 3 - s.Cluster.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy = &v1alpha1.ControlPlaneUpgradeRolloutStrategy{ - Type: "RollingUpdate", - RollingUpdate: v1alpha1.ControlPlaneRollingUpdateParams{ - MaxSurge: 1, - }, - } - s.VersionsBundles["1.19"] = versionsBundle - s.Cluster.Spec.ExternalEtcdConfiguration = &v1alpha1.ExternalEtcdConfiguration{Count: 3} - s.Cluster.Spec.WorkerNodeGroupConfigurations = []v1alpha1.WorkerNodeGroupConfiguration{{ - Count: ptr.Int(3), - MachineGroupRef: &v1alpha1.Ref{Name: "test-cluster"}, - Name: "md-0", - UpgradeRolloutStrategy: &v1alpha1.WorkerNodesUpgradeRolloutStrategy{ - Type: "RollingUpdate", - RollingUpdate: v1alpha1.WorkerNodesRollingUpdateParams{ - MaxSurge: 1, - MaxUnavailable: 0, - }, - }, - }} - }), - wantCPFile: "testdata/valid_deployment_cp_rollout_expected.yaml", - wantMDFile: "testdata/valid_deployment_md_rollout_expected.yaml", - }, } for _, tt := range tests { diff --git a/pkg/providers/docker/testdata/valid_deployment_cp_rollout_expected.yaml b/pkg/providers/docker/testdata/valid_deployment_cp_rollout_expected.yaml deleted file mode 100644 index 43e3ca23ca3c3..0000000000000 --- a/pkg/providers/docker/testdata/valid_deployment_cp_rollout_expected.yaml +++ /dev/null @@ -1,316 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: test-cluster - namespace: eksa-system -spec: - clusterNetwork: - pods: - cidrBlocks: [192.168.0.0/16] - serviceDomain: cluster.local - services: - cidrBlocks: [10.128.0.0/12] - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: test-cluster - namespace: eksa-system - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerCluster - name: test-cluster - namespace: eksa-system - managedExternalEtcdRef: - apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 - kind: EtcdadmCluster - name: test-cluster-etcd - namespace: eksa-system ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerCluster -metadata: - name: test-cluster - namespace: eksa-system -spec: - loadBalancer: - imageRepository: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind - imageTag: v0.11.1-eks-a-v0.0.0-dev-build.1464 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: test-cluster-control-plane-template-1234567890000 - namespace: eksa-system -spec: - template: - spec: - extraMounts: - - containerPath: /var/run/docker.sock - hostPath: /var/run/docker.sock - customImage: public.ecr.aws/eks-distro/kubernetes-sigs/kind/node:v1.18.16-eks-1-18-4-216edda697a37f8bf16651af6c23b7e2bb7ef42f-62681885fe3a97ee4f2b110cc277e084e71230fa ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: test-cluster - namespace: eksa-system -spec: - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: test-cluster-control-plane-template-1234567890000 - namespace: eksa-system - kubeadmConfigSpec: - clusterConfiguration: - imageRepository: public.ecr.aws/eks-distro/kubernetes - etcd: - external: - endpoints: [] - caFile: "/etc/kubernetes/pki/etcd/ca.crt" - certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt" - keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key" - dns: - imageRepository: public.ecr.aws/eks-distro/coredns - imageTag: v1.8.0-eks-1-19-2 - apiServer: - certSANs: - - localhost - - 127.0.0.1 - extraArgs: - audit-policy-file: /etc/kubernetes/audit-policy.yaml - audit-log-path: /var/log/kubernetes/api-audit.log - audit-log-maxage: "30" - audit-log-maxbackup: "10" - audit-log-maxsize: "512" - profiling: "false" - tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - extraVolumes: - - hostPath: /etc/kubernetes/audit-policy.yaml - mountPath: /etc/kubernetes/audit-policy.yaml - name: audit-policy - pathType: File - readOnly: true - - hostPath: /var/log/kubernetes - mountPath: /var/log/kubernetes - name: audit-log-dir - pathType: DirectoryOrCreate - readOnly: false - controllerManager: - extraArgs: - enable-hostpath-provisioner: "true" - profiling: "false" - tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - scheduler: - extraArgs: - profiling: "false" - tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - files: - - content: | - apiVersion: audit.k8s.io/v1beta1 - kind: Policy - rules: - # Log aws-auth configmap changes - - level: RequestResponse - namespaces: ["kube-system"] - verbs: ["update", "patch", "delete"] - resources: - - group: "" # core - resources: ["configmaps"] - resourceNames: ["aws-auth"] - omitStages: - - "RequestReceived" - # The following requests were manually identified as high-volume and low-risk, - # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - - level: None - users: ["kubelet"] # legacy kubelet identity - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - userGroups: ["system:nodes"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - users: - - system:kube-controller-manager - - system:kube-scheduler - - system:serviceaccount:kube-system:endpoint-controller - verbs: ["get", "update"] - namespaces: ["kube-system"] - resources: - - group: "" # core - resources: ["endpoints"] - - level: None - users: ["system:apiserver"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["namespaces", "namespaces/status", "namespaces/finalize"] - # Don't log HPA fetching metrics. - - level: None - users: - - system:kube-controller-manager - verbs: ["get", "list"] - resources: - - group: "metrics.k8s.io" - # Don't log these read-only URLs. - - level: None - nonResourceURLs: - - /healthz* - - /version - - /swagger* - # Don't log events requests. - - level: None - resources: - - group: "" # core - resources: ["events"] - # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes - - level: Request - users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] - verbs: ["update","patch"] - resources: - - group: "" # core - resources: ["nodes/status", "pods/status"] - omitStages: - - "RequestReceived" - - level: Request - userGroups: ["system:nodes"] - verbs: ["update","patch"] - resources: - - group: "" # core - resources: ["nodes/status", "pods/status"] - omitStages: - - "RequestReceived" - # deletecollection calls can be large, don't log responses for expected namespace deletions - - level: Request - users: ["system:serviceaccount:kube-system:namespace-controller"] - verbs: ["deletecollection"] - omitStages: - - "RequestReceived" - # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, - # so only log at the Metadata level. - - level: Metadata - resources: - - group: "" # core - resources: ["secrets", "configmaps"] - - group: authentication.k8s.io - resources: ["tokenreviews"] - omitStages: - - "RequestReceived" - - level: Request - resources: - - group: "" - resources: ["serviceaccounts/token"] - # Get repsonses can be large; skip them. - - level: Request - verbs: ["get", "list", "watch"] - resources: - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics.k8s.io" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "scheduling.k8s.io" - - group: "settings.k8s.io" - - group: "storage.k8s.io" - omitStages: - - "RequestReceived" - # Default level for known APIs - - level: RequestResponse - resources: - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics.k8s.io" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "scheduling.k8s.io" - - group: "settings.k8s.io" - - group: "storage.k8s.io" - omitStages: - - "RequestReceived" - # Default level for all other requests. - - level: Metadata - omitStages: - - "RequestReceived" - owner: root:root - path: /etc/kubernetes/audit-policy.yaml - initConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cgroup-driver: cgroupfs - tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - joinConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cgroup-driver: cgroupfs - tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - replicas: 3 - rolloutStrategy: - rollingUpdate: - maxSurge: 1 - version: v1.19.6-eks-1-19-2 ---- -kind: EtcdadmCluster -apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 -metadata: - name: test-cluster-etcd - namespace: eksa-system -spec: - replicas: 3 - etcdadmConfigSpec: - etcdadmBuiltin: true - cloudInitConfig: - version: 3.4.14 - cipherSuites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: test-cluster-etcd-template-1234567890000 - namespace: eksa-system ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: test-cluster-etcd-template-1234567890000 - namespace: eksa-system -spec: - template: - spec: - extraMounts: - - containerPath: /var/run/docker.sock - hostPath: /var/run/docker.sock - customImage: public.ecr.aws/eks-distro/kubernetes-sigs/kind/node:v1.18.16-eks-1-18-4-216edda697a37f8bf16651af6c23b7e2bb7ef42f-62681885fe3a97ee4f2b110cc277e084e71230fa diff --git a/pkg/providers/docker/testdata/valid_deployment_md_rollout_expected.yaml b/pkg/providers/docker/testdata/valid_deployment_md_rollout_expected.yaml deleted file mode 100644 index 2cffff3bac03a..0000000000000 --- a/pkg/providers/docker/testdata/valid_deployment_md_rollout_expected.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: test-cluster-md-0-template-1234567890000 - namespace: eksa-system -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - taints: [] - kubeletExtraArgs: - eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cgroup-driver: cgroupfs - tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: test-cluster-md-0 - namespace: eksa-system -spec: - clusterName: test-cluster - replicas: 3 - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: test-cluster-md-0-template-1234567890000 - namespace: eksa-system - clusterName: test-cluster - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: test-cluster-md-0-1234567890000 - namespace: eksa-system - version: v1.19.6-eks-1-19-2 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerMachineTemplate -metadata: - name: test-cluster-md-0-1234567890000 - namespace: eksa-system -spec: - template: - spec: - extraMounts: - - containerPath: /var/run/docker.sock - hostPath: /var/run/docker.sock - customImage: public.ecr.aws/eks-distro/kubernetes-sigs/kind/node:v1.18.16-eks-1-18-4-216edda697a37f8bf16651af6c23b7e2bb7ef42f-62681885fe3a97ee4f2b110cc277e084e71230fa - ---- diff --git a/pkg/providers/docker/workers_test.go b/pkg/providers/docker/workers_test.go index 193150fa1d2ab..b3a5866375206 100644 --- a/pkg/providers/docker/workers_test.go +++ b/pkg/providers/docker/workers_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" dockerv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" @@ -15,6 +16,7 @@ import ( "github.com/aws/eks-anywhere/internal/test" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/clients/kubernetes" + "github.com/aws/eks-anywhere/pkg/cluster" "github.com/aws/eks-anywhere/pkg/clusterapi" "github.com/aws/eks-anywhere/pkg/controller/clientutil" "github.com/aws/eks-anywhere/pkg/providers/docker" @@ -334,6 +336,49 @@ func TestWorkersSpecRegistryMirrorConfiguration(t *testing.T) { } } +func TestWorkersSpecUpgradeRolloutStrategy(t *testing.T) { + g := NewWithT(t) + logger := test.NewNullLogger() + ctx := context.Background() + spec := testClusterSpec(func(s *cluster.Spec) { + s.Cluster.Spec.WorkerNodeGroupConfigurations = []anywherev1.WorkerNodeGroupConfiguration{ + { + Count: ptr.Int(3), + MachineGroupRef: &anywherev1.Ref{Name: "test"}, + Name: "md-0", + UpgradeRolloutStrategy: &anywherev1.WorkerNodesUpgradeRolloutStrategy{ + RollingUpdate: anywherev1.WorkerNodesRollingUpdateParams{ + MaxSurge: 1, + MaxUnavailable: 0, + }, + }, + }, + } + }) + client := test.NewFakeKubeClient() + + workers, err := docker.WorkersSpec(ctx, logger, client, spec) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(workers).NotTo(BeNil()) + g.Expect(workers.Groups).To(HaveLen(1)) + g.Expect(workers.Groups).To(ConsistOf( + clusterapi.WorkerGroup[*dockerv1.DockerMachineTemplate]{ + KubeadmConfigTemplate: kubeadmConfigTemplate(), + MachineDeployment: machineDeployment(func(m *clusterv1.MachineDeployment) { + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(0) + m.Spec.Strategy = &clusterv1.MachineDeploymentStrategy{ + RollingUpdate: &clusterv1.MachineRollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + } + }), + ProviderMachineTemplate: dockerMachineTemplate("test-md-0-1"), + }, + )) +} + func kubeadmConfigTemplate(opts ...func(*bootstrapv1.KubeadmConfigTemplate)) *bootstrapv1.KubeadmConfigTemplate { o := &bootstrapv1.KubeadmConfigTemplate{ TypeMeta: metav1.TypeMeta{