From 2a746e60e8dfb7529e6acc7ce5adf714fd063fba Mon Sep 17 00:00:00 2001 From: Jeff Banks Date: Fri, 12 Nov 2021 15:57:18 -0600 Subject: [PATCH] Change /etc/reaper to /etc/cassandra-reaper (#84) --- .gitignore | 2 +- Makefile | 5 +- api/v1alpha1/reaper_types.go | 5 +- api/v1alpha1/zz_generated.deepcopy.go | 2 + .../reaper.cassandra-reaper.io_reapers.yaml | 90 ++++++++++++- controllers/reaper_controller_test.go | 11 ++ pkg/reconcile/reconcilers.go | 45 ++++++- pkg/reconcile/reconcilers_test.go | 125 ++++++++++++++++-- 8 files changed, 264 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 8676887..ed5604b 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,6 @@ tags # forks for running tests with custom images test/config/deploy_reaper_test/overlays/forks/ - kind-logs*/ build/ + diff --git a/Makefile b/Makefile index ae9d9c8..feeb4b2 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ ORG?=k8ssandra PROJECT=reaper-operator REG?=docker.io +# Change from default of 'kind' as needed +KIND_CLUSTER_NAME=kind + BRANCH=$(shell git rev-parse --abbrev-ref HEAD) REV=$(shell git rev-parse --short=8 HEAD) @@ -121,7 +124,7 @@ docker-push: docker push ${LATEST_IMAGE} kind-load-image: - kind load docker-image ${LATEST_IMAGE} + kind load docker-image ${LATEST_IMAGE} --name ${KIND_CLUSTER_NAME} # find or download controller-gen # download controller-gen if necessary diff --git a/api/v1alpha1/reaper_types.go b/api/v1alpha1/reaper_types.go index a156f4a..9c49d58 100644 --- a/api/v1alpha1/reaper_types.go +++ b/api/v1alpha1/reaper_types.go @@ -113,9 +113,12 @@ type ReaperSpec struct { // Tolerations applied to the Reaper pods Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - // InitContainerConfig encapsulates configs applied to the Reaper init container + // SchemaInitContainerConfig encapsulates settings applied to the Reaper schema init container SchemaInitContainerConfig InitContainerConfig `json:"schemaInitContainerConfig,omitempty" yaml:"schemaInitContainerConfig,omitempty"` + // ConfigInitContainerConfig encapsulates settings applied to the Reaper config init container + ConfigInitContainerConfig InitContainerConfig `json:"configInitContainerConfig,omitempty" yaml:"configInitContainerConfig,omitempty"` + // SecurityContext applied to the Reaper non-init container SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 93c04b6..e31ea91 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -178,6 +179,7 @@ func (in *ReaperSpec) DeepCopyInto(out *ReaperSpec) { } } in.SchemaInitContainerConfig.DeepCopyInto(&out.SchemaInitContainerConfig) + in.ConfigInitContainerConfig.DeepCopyInto(&out.ConfigInitContainerConfig) if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext *out = new(v1.SecurityContext) diff --git a/config/crd/bases/reaper.cassandra-reaper.io_reapers.yaml b/config/crd/bases/reaper.cassandra-reaper.io_reapers.yaml index d55e113..09d9ff2 100644 --- a/config/crd/bases/reaper.cassandra-reaper.io_reapers.yaml +++ b/config/crd/bases/reaper.cassandra-reaper.io_reapers.yaml @@ -375,6 +375,94 @@ spec: type: array type: object type: object + configInitContainerConfig: + description: ConfigInitContainerConfig encapsulates settings applied to the Reaper config init container + properties: + securityContext: + description: SecurityContext applied to a Reaper init container + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + type: object image: type: string imagePullPolicy: @@ -465,7 +553,7 @@ spec: type: object type: object schemaInitContainerConfig: - description: InitContainerConfig encapsulates configs applied to the Reaper init container + description: SchemaInitContainerConfig encapsulates settings applied to the Reaper schema init container properties: securityContext: description: SecurityContext applied to a Reaper init container diff --git a/controllers/reaper_controller_test.go b/controllers/reaper_controller_test.go index 8012046..12bfe11 100644 --- a/controllers/reaper_controller_test.go +++ b/controllers/reaper_controller_test.go @@ -353,6 +353,17 @@ var _ = Describe("Reaper controller", func() { Expect(envVars[len(envVars)-2].ValueFrom.SecretKeyRef.Key).To(Equal("password")) Expect(envVars[len(envVars)-1].Name).To(Equal("REAPER_CASS_AUTH_ENABLED")) Expect(envVars[len(envVars)-1].Value).To(Equal("true")) + + // Schema init env var check, note: config init doesn't currently utilize these vars + schemaInitEnvVars := deployment.Spec.Template.Spec.InitContainers[1].Env + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-3].Name).To(Equal("REAPER_CASS_AUTH_USERNAME")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-3].ValueFrom.SecretKeyRef.LocalObjectReference.Name).To(Equal("top-secret-cass")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-3].ValueFrom.SecretKeyRef.Key).To(Equal("username")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-2].Name).To(Equal("REAPER_CASS_AUTH_PASSWORD")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-2].ValueFrom.SecretKeyRef.LocalObjectReference.Name).To(Equal("top-secret-cass")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-2].ValueFrom.SecretKeyRef.Key).To(Equal("password")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-1].Name).To(Equal("REAPER_CASS_AUTH_ENABLED")) + Expect(schemaInitEnvVars[len(schemaInitEnvVars)-1].Value).To(Equal("true")) }) }) diff --git a/pkg/reconcile/reconcilers.go b/pkg/reconcile/reconcilers.go index ecc3149..0123acd 100644 --- a/pkg/reconcile/reconcilers.go +++ b/pkg/reconcile/reconcilers.go @@ -325,6 +325,9 @@ func (r *defaultReconciler) ReconcileDeployment(ctx context.Context, req ReaperR deployment.Spec.RevisionHistoryLimit = desiredDeployment.Spec.RevisionHistoryLimit deployment.Spec.Strategy = desiredDeployment.Spec.Strategy + deployment.Spec.Template.Spec.Volumes = desiredDeployment.Spec.Template.Spec.Volumes + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts + if err = r.Update(ctx, deployment); err != nil { req.Logger.Error(err, "failed to update deployment", "deployment", deployment) } @@ -401,13 +404,14 @@ func (r *defaultReconciler) buildNewDeployment(req ReaperRequest) (*appsv1.Deplo } func addAuthEnvVars(deployment *appsv1.Deployment, vars ...*corev1.EnvVar) { - initEnvVars := deployment.Spec.Template.Spec.InitContainers[0].Env + schemaInitEnvVars := deployment.Spec.Template.Spec.InitContainers[1].Env envVars := deployment.Spec.Template.Spec.Containers[0].Env + for _, v := range vars { - initEnvVars = append(initEnvVars, *v) + schemaInitEnvVars = append(schemaInitEnvVars, *v) envVars = append(envVars, *v) } - deployment.Spec.Template.Spec.InitContainers[0].Env = initEnvVars + deployment.Spec.Template.Spec.InitContainers[1].Env = schemaInitEnvVars deployment.Spec.Template.Spec.Containers[0].Env = envVars } @@ -506,7 +510,6 @@ func newDeployment(reaper *api.Reaper, cassDcService string) *appsv1.Deployment } } } - return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: reaper.Namespace, @@ -525,6 +528,21 @@ func newDeployment(reaper *api.Reaper, cassDcService string) *appsv1.Deployment Spec: corev1.PodSpec{ Affinity: reaper.Spec.Affinity, InitContainers: []corev1.Container{ + { + Name: "reaper-config-init", + ImagePullPolicy: corev1.PullPolicy(reaper.Spec.ImagePullPolicy), + Image: reaper.Spec.Image, + SecurityContext: reaper.Spec.ConfigInitContainerConfig.SecurityContext, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "cp -r /etc/cassandra-reaper/* /reaper-base-config/"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "reaper-config", + ReadOnly: false, + MountPath: "/reaper-base-config/", + }, + }, + }, { Name: "reaper-schema-init", ImagePullPolicy: corev1.PullPolicy(reaper.Spec.ImagePullPolicy), @@ -532,6 +550,13 @@ func newDeployment(reaper *api.Reaper, cassDcService string) *appsv1.Deployment SecurityContext: reaper.Spec.SchemaInitContainerConfig.SecurityContext, Env: envVars, Args: []string{"schema-migration"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "reaper-config", + ReadOnly: false, + MountPath: "/etc/cassandra-reaper", + }, + }, }, }, Containers: []corev1.Container{ @@ -539,6 +564,13 @@ func newDeployment(reaper *api.Reaper, cassDcService string) *appsv1.Deployment Name: "reaper", ImagePullPolicy: corev1.PullPolicy(reaper.Spec.ImagePullPolicy), Image: reaper.Spec.Image, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "reaper-config", + ReadOnly: true, + MountPath: "/etc/cassandra-reaper", + }, + }, SecurityContext: reaper.Spec.SecurityContext, Ports: []corev1.ContainerPort{ { @@ -560,6 +592,11 @@ func newDeployment(reaper *api.Reaper, cassDcService string) *appsv1.Deployment ServiceAccountName: reaper.Spec.ServiceAccountName, Tolerations: reaper.Spec.Tolerations, SecurityContext: reaper.Spec.PodSecurityContext, + Volumes: []corev1.Volume{ + { + Name: "reaper-config", + }, + }, }, }, }, diff --git a/pkg/reconcile/reconcilers_test.go b/pkg/reconcile/reconcilers_test.go index 7fec5ae..9fb6728 100644 --- a/pkg/reconcile/reconcilers_test.go +++ b/pkg/reconcile/reconcilers_test.go @@ -109,12 +109,32 @@ func TestNewDeployment(t *testing.T) { }, }) - assert.Equal(1, len(podSpec.InitContainers)) + assert.Equal(2, len(podSpec.InitContainers)) - initContainer := podSpec.InitContainers[0] - assert.Equal(image, initContainer.Image) - assert.Equal(corev1.PullAlways, initContainer.ImagePullPolicy) - assert.ElementsMatch(initContainer.Env, []corev1.EnvVar{ + // Reaper configuration init-container + initContainerConfigInit := podSpec.InitContainers[0] + expectedVolMount := []corev1.VolumeMount{ + { + Name: "reaper-config", + ReadOnly: false, + MountPath: "/reaper-base-config/", + }, + } + assert.Equal("reaper-config-init", initContainerConfigInit.Name) + assert.Equal(image, initContainerConfigInit.Image) + assert.Equal([]string{"/bin/sh"}, initContainerConfigInit.Command) + assert.Equal([]string{"-c", "cp -r /etc/cassandra-reaper/* /reaper-base-config/"}, initContainerConfigInit.Args) + + assert.Equal(corev1.PullAlways, initContainerConfigInit.ImagePullPolicy) + assert.ElementsMatch(expectedVolMount, initContainerConfigInit.VolumeMounts) + + // Schema initialization init-container + initContainerSchemaInit := podSpec.InitContainers[1] + assert.Equal(image, initContainerSchemaInit.Image) + + assert.Equal("reaper-schema-init", initContainerSchemaInit.Name) + assert.Equal(corev1.PullAlways, initContainerSchemaInit.ImagePullPolicy) + assert.ElementsMatch(initContainerSchemaInit.Env, []corev1.EnvVar{ { Name: "REAPER_STORAGE_TYPE", Value: "cassandra", @@ -136,9 +156,9 @@ func TestNewDeployment(t *testing.T) { Value: "true", }, }) + assert.ElementsMatch(initContainerSchemaInit.Args, []string{"schema-migration"}) - assert.ElementsMatch(initContainer.Args, []string{"schema-migration"}) - + // ServerConfig AutoScheduling reaper.Spec.ServerConfig.AutoScheduling = &api.AutoScheduler{ Enabled: false, InitialDelay: "PT10S", @@ -277,25 +297,37 @@ func TestContainerSecurityContext(t *testing.T) { func TestSchemaInitContainerSecurityContext(t *testing.T) { image := "test/reaper:latest" readOnlyRootFilesystemOverride := true - initContainerSecurityContext := &corev1.SecurityContext{ + schemaInitSecurityContext := &corev1.SecurityContext{ ReadOnlyRootFilesystem: &readOnlyRootFilesystemOverride, } nonInitContainerSecurityContext := &corev1.SecurityContext{ ReadOnlyRootFilesystem: &readOnlyRootFilesystemOverride, } + configInitSecurityContext := &corev1.SecurityContext{ + ReadOnlyRootFilesystem: &readOnlyRootFilesystemOverride, + } reaper := newReaperWithCassandraBackend("service-test", "test-reaper") reaper.Spec.Image = image reaper.Spec.SecurityContext = nonInitContainerSecurityContext - reaper.Spec.SchemaInitContainerConfig.SecurityContext = initContainerSecurityContext + reaper.Spec.SchemaInitContainerConfig.SecurityContext = schemaInitSecurityContext + reaper.Spec.ConfigInitContainerConfig.SecurityContext = configInitSecurityContext deployment := newDeployment(reaper, "target-datacenter-service") podSpec := deployment.Spec.Template.Spec - assert.Equal(t, podSpec.InitContainers[0].Name, "reaper-schema-init") - assert.True(t, len(podSpec.InitContainers) == 1, "Expected a single schema init container to exist") - same := assert.ObjectsAreEqualValues(initContainerSecurityContext, podSpec.InitContainers[0].SecurityContext) - assert.True(t, same, "securityContext does not match for schema init container") + assert.True(t, len(podSpec.InitContainers) == 2, "Expected two init containers to exist") + assert.Equal(t, podSpec.InitContainers[0].Name, "reaper-config-init") + assert.Equal(t, podSpec.InitContainers[1].Name, "reaper-schema-init") + + schemaInitSecurityCtxNotMatch := assert.ObjectsAreEqualValues(schemaInitSecurityContext, + podSpec.InitContainers[0].SecurityContext) + + configInitSecurityCtxNotMatch := assert.ObjectsAreEqualValues(configInitSecurityContext, + podSpec.InitContainers[1].SecurityContext) + + assert.True(t, schemaInitSecurityCtxNotMatch, "securityContext does not match for schema init container") + assert.True(t, configInitSecurityCtxNotMatch, "securityContext does not match for config init container") } func TestPodSecurityContext(t *testing.T) { @@ -315,6 +347,73 @@ func TestPodSecurityContext(t *testing.T) { assert.True(t, same, "podSecurityContext expected at pod level") } +func TestVolumes(t *testing.T) { + image := "test/reaper:latest" + volumes := []corev1.Volume{ + { + Name: "reaper-config", + }, + } + + reaper := newReaperWithCassandraBackend("service-test", "test-reaper") + reaper.Spec.Image = image + + deployment := newDeployment(reaper, "target-datacenter-service") + assert.ElementsMatch(t, volumes, deployment.Spec.Template.Spec.Volumes) +} + +func TestSchemaInitContainerVolumesMounts(t *testing.T) { + image := "test/reaper:latest" + volumes := []corev1.Volume{ + { + Name: "reaper-config", + }, + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: "reaper-config", + ReadOnly: false, + MountPath: "/etc/cassandra-reaper", + }, + } + + reaper := newReaperWithCassandraBackend("service-test", "test-reaper") + reaper.Spec.Image = image + + deployment := newDeployment(reaper, "target-datacenter-service") + + assert.ElementsMatch(t, volumes, deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, "reaper-schema-init", deployment.Spec.Template.Spec.InitContainers[1].Name) + assert.ElementsMatch(t, volumeMounts, deployment.Spec.Template.Spec.InitContainers[1].VolumeMounts) +} + +func TestConfigInitContainerVolumesMounts(t *testing.T) { + + image := "test/reaper:latest" + volumes := []corev1.Volume{ + { + Name: "reaper-config", + }, + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: "reaper-config", + ReadOnly: false, + MountPath: "/reaper-base-config/", + }, + } + + reaper := newReaperWithCassandraBackend("service-test", "test-reaper") + reaper.Spec.Image = image + + deployment := newDeployment(reaper, "target-datacenter-service") + assert.ElementsMatch(t, volumes, deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, "reaper-config-init", deployment.Spec.Template.Spec.InitContainers[0].Name) + assert.ElementsMatch(t, volumeMounts, deployment.Spec.Template.Spec.InitContainers[0].VolumeMounts) +} + func TestReaperDeploymentStrategy(t *testing.T) { // Test to ensure that if we set DeploymentStrategy to Rolling then it gets set back to Recreate by reconcile() // Reaper test resource