diff --git a/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml b/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml index d0b3c741..3ebf125a 100644 --- a/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml +++ b/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml @@ -29,6 +29,12 @@ spec: type: object spec: properties: + backupSecretVolumeSource: + type: object + cassandraEnv: + items: + type: object + type: array cassandraImage: type: string cluster: @@ -46,15 +52,25 @@ spec: nodes: format: int32 type: integer + privilegedSupported: + type: boolean prometheusSupport: type: boolean resources: type: object + sidecarEnv: + items: + type: object + type: array serviceAccountName: description: ServiceAccount to assign to pods created by the operator type: string sidecarImage: type: string + userConfigMapVolumeSource: + type: object + userSecretVolumeSource: + type: object required: - nodes - cassandraImage diff --git a/doc/backup_restore.md b/doc/backup_restore.md index 97fc7ab5..5107be52 100644 --- a/doc/backup_restore.md +++ b/doc/backup_restore.md @@ -22,7 +22,7 @@ You can inspect the secret created via `kubectl describe secrets/awsbackuptest` Create a `CassandraDataCenter` CRD that injects the secret as environment variables that matches the AWS client libraries expected env variables: ```yaml - env: + sidecarEnv: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -65,7 +65,7 @@ spec: resources: requests: storage: 100Mi - env: + sidecarEnv: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -85,6 +85,9 @@ spec: To create a cluster using this yaml file use `kubectl apply -f myBackupCluster.yaml` ## Configuring GCP Object Storage via environment variables +The backup credentials will be added to the sidecar container at the `/tmp/backup-creds` location. +Use this location to set GOOGLE_APPLICATION_CREDENTIALS environment variable to the key json file stored in the secret. + First create a secret in kubernetes to hold a Google service account token/file (assuming they are stored in files named access and secret respectively). `kubectl create secret generic gcp-auth-reference --from-file=my_service_key.json` @@ -118,14 +121,14 @@ spec: resources: requests: storage: 100Mi - userSecretSource: + backupSecretVolumeSource: name: gcp-auth-reference items: - key: my_service_key.json path: my_service_key.json - env: + sidecarEnv: - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/tmp/user-secret/my_service_key.json" + value: "/tmp/backup-creds/my_service_key.json" - name: GOOGLE_CLOUD_PROJECT value: "cassandra-operator" - name: BUCKET_NAME diff --git a/examples/go/example-datacenter.yaml b/examples/go/example-datacenter.yaml index 8b3a5921..ba138637 100644 --- a/examples/go/example-datacenter.yaml +++ b/examples/go/example-datacenter.yaml @@ -10,6 +10,27 @@ spec: cassandraImage: "gcr.io/cassandra-operator/cassandra:3.11.3" sidecarImage: "gcr.io/cassandra-operator/cassandra-sidecar:latest" imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: regcred + backupSecretVolumeSource: + # example from doc/backup_restore.md + secretName: gcp-auth-reference + # type is a workaround for https://github.com/kubernetes/kubernetes/issues/68466 + type: array + items: + - key: my-service-key.json + path: my-service-key.json + sidecarEnv: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/tmp/backup-creds/my-service-key.json" + userConfigMapVolumeSource: + # example from doc/providers/pks.md + # the name of the ConfigMap + name: concurrent-data + # ConfigMap keys -> file paths (relative to /etc/cassandra) + items: + - key: 100-concurrent-yaml + path: cassandra.yaml.d/100-concurrent.yaml resources: limits: memory: 1Gi diff --git a/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter_types.go b/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter_types.go index 64a81d51..13017961 100644 --- a/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter_types.go +++ b/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter_types.go @@ -10,19 +10,21 @@ import ( type CassandraDataCenterSpec struct { // Cluster is either a string or v1.LocalObjectReference //Cluster interface{} `json:"cluster,omitempty"` - Cluster string `json:"cluster,omitempty"` - Nodes int32 `json:"nodes"` - CassandraImage string `json:"cassandraImage"` - SidecarImage string `json:"sidecarImage"` - ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy"` - ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - - Resources v1.ResourceRequirements `json:"resources"` - - DataVolumeClaimSpec v1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"` - - PrometheusSupport bool `json:"prometheusSupport"` - + Cluster string `json:"cluster,omitempty"` + Nodes int32 `json:"nodes"` + CassandraImage string `json:"cassandraImage"` + SidecarImage string `json:"sidecarImage"` + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy"` + ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + BackupSecretVolumeSource *v1.SecretVolumeSource `json:"backupSecretVolumeSource,omitempty"` + UserSecretVolumeSource *v1.SecretVolumeSource `json:"userSecretVolumeSource,omitempty"` + UserConfigMapVolumeSource *v1.ConfigMapVolumeSource `json:"userConfigMapVolumeSource,omitempty"` + Resources v1.ResourceRequirements `json:"resources"` + DataVolumeClaimSpec v1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"` + PrivilegedSupported bool `json:"privilegedSupported,omitempty"` + PrometheusSupport bool `json:"prometheusSupport"` + SidecarEnv []v1.EnvVar `json:"sidecarEnv,omitempty"` + CassandraEnv []v1.EnvVar `json:"cassandraEnv,omitempty"` // ServiceAccount to assign to pods created by the operator ServiceAccountName string `json:"serviceAccountName,omitempty"` } diff --git a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go index f08093d7..8f7d7b9e 100644 --- a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go @@ -283,8 +283,37 @@ func (in *CassandraDataCenterSpec) DeepCopyInto(out *CassandraDataCenterSpec) { *out = make([]v1.LocalObjectReference, len(*in)) copy(*out, *in) } + if in.BackupSecretVolumeSource != nil { + in, out := &in.BackupSecretVolumeSource, &out.BackupSecretVolumeSource + *out = new(v1.SecretVolumeSource) + (*in).DeepCopyInto(*out) + } + if in.UserSecretVolumeSource != nil { + in, out := &in.UserSecretVolumeSource, &out.UserSecretVolumeSource + *out = new(v1.SecretVolumeSource) + (*in).DeepCopyInto(*out) + } + if in.UserConfigMapVolumeSource != nil { + in, out := &in.UserConfigMapVolumeSource, &out.UserConfigMapVolumeSource + *out = new(v1.ConfigMapVolumeSource) + (*in).DeepCopyInto(*out) + } in.Resources.DeepCopyInto(&out.Resources) in.DataVolumeClaimSpec.DeepCopyInto(&out.DataVolumeClaimSpec) + if in.SidecarEnv != nil { + in, out := &in.SidecarEnv, &out.SidecarEnv + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.CassandraEnv != nil { + in, out := &in.CassandraEnv, &out.CassandraEnv + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go index f4319dc3..728da4cd 100644 --- a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go +++ b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go @@ -309,6 +309,21 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm }, }, }, + "backupSecretVolumeSource": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.SecretVolumeSource"), + }, + }, + "userSecretVolumeSource": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.SecretVolumeSource"), + }, + }, + "userConfigMapVolumeSource": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ConfigMapVolumeSource"), + }, + }, "resources": { SchemaProps: spec.SchemaProps{ Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), @@ -319,12 +334,42 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm Ref: ref("k8s.io/api/core/v1.PersistentVolumeClaimSpec"), }, }, + "privilegedSupported": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, "prometheusSupport": { SchemaProps: spec.SchemaProps{ Type: []string{"boolean"}, Format: "", }, }, + "sidecarEnv": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.EnvVar"), + }, + }, + }, + }, + }, + "cassandraEnv": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.EnvVar"), + }, + }, + }, + }, + }, "serviceAccountName": { SchemaProps: spec.SchemaProps{ Description: "ServiceAccount to assign to pods created by the operator", @@ -337,7 +382,7 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm }, }, Dependencies: []string{ - "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaimSpec", "k8s.io/api/core/v1.ResourceRequirements"}, + "k8s.io/api/core/v1.ConfigMapVolumeSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaimSpec", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecretVolumeSource"}, } } diff --git a/pkg/controller/cassandradatacenter/configmap.go b/pkg/controller/cassandradatacenter/configmap.go index e1de0696..97c48ffd 100644 --- a/pkg/controller/cassandradatacenter/configmap.go +++ b/pkg/controller/cassandradatacenter/configmap.go @@ -38,6 +38,7 @@ func createOrUpdateOperatorConfigMap(rctx *reconciliationRequestContext, seedNod addPrometheusSupport(rctx.cdc, addFileFn) + if err := controllerutil.SetControllerReference(rctx.cdc, configMap, rctx.scheme); err != nil { return err } diff --git a/pkg/controller/cassandradatacenter/statefulset.go b/pkg/controller/cassandradatacenter/statefulset.go index b08c2348..2a0aa2eb 100644 --- a/pkg/controller/cassandradatacenter/statefulset.go +++ b/pkg/controller/cassandradatacenter/statefulset.go @@ -19,14 +19,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -const DataVolumeMountPath = "/var/lib/cassandra" - -const SidecarApiPort = 4567 - -var sidecarClientOptions = sidecar.ClientOptions{ - Port: SidecarApiPort, - Secure: false, -} +const ( + DataVolumeMountPath = "/var/lib/cassandra" + OperatorConfigVolumeMountPath = "/tmp/operator-config" + UserConfigVolumeMountPath = "/tmp/user-config" + UserSecretVolumeMountPath = "/tmp/user-secret" + BackupSecretVolumeMountPath = "/tmp/backup-secret" +) func createOrUpdateStatefulSet(rctx *reconciliationRequestContext, configVolume *corev1.Volume) (*v1beta2.StatefulSet, error) { statefulSet := &v1beta2.StatefulSet{ObjectMeta: DataCenterResourceMetadata(rctx.cdc)} @@ -40,11 +39,13 @@ func createOrUpdateStatefulSet(rctx *reconciliationRequestContext, configVolume } dataVolumeClaim := newDataVolumeClaim(&rctx.cdc.Spec.DataVolumeClaimSpec) - podInfoVolume := newPodInfoVolume() + backupSecretVolume := newBackupSecretVolume(rctx) + userSecretVolume := newUserSecretVolume(rctx) + userConfigVolume := newUserConfigVolume(rctx) - cassandraContainer := newCassandraContainer(rctx.cdc, dataVolumeClaim, configVolume) - sidecarContainer := newSidecarContainer(rctx.cdc, dataVolumeClaim, podInfoVolume) + cassandraContainer := newCassandraContainer(rctx.cdc, dataVolumeClaim, configVolume, userSecretVolume, userConfigVolume) + sidecarContainer := newSidecarContainer(rctx.cdc, dataVolumeClaim, podInfoVolume, backupSecretVolume) sysctlLimitsContainer := newSysctlLimitsContainer(rctx.cdc) @@ -53,6 +54,18 @@ func createOrUpdateStatefulSet(rctx *reconciliationRequestContext, configVolume []corev1.Container{*cassandraContainer, *sidecarContainer}, []corev1.Container{*sysctlLimitsContainer}) + if backupSecretVolume != nil { + podSpec.Volumes = append(podSpec.Volumes, *backupSecretVolume) + } + + if userSecretVolume != nil { + podSpec.Volumes = append(podSpec.Volumes, *userSecretVolume) + } + + if userConfigVolume != nil { + podSpec.Volumes = append(podSpec.Volumes, *userConfigVolume) + } + statefulSetSpec := newStatefulSetSpec(rctx.cdc, podSpec, dataVolumeClaim) if statefulSet.CreationTimestamp.IsZero() { @@ -102,9 +115,7 @@ func newPodSpec(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, volumes []co return podSpec } -func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, configVolume *corev1.Volume) *corev1.Container { - const OperatorConfigVolumeMountPath = "/tmp/operator-config" - +func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, configVolume *corev1.Volume, userSecretVolume, userConfigVolume *corev1.Volume) *corev1.Container { container := &corev1.Container{ Name: "cassandra", Image: cdc.Spec.CassandraImage, @@ -112,6 +123,7 @@ func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, d Args: []string{OperatorConfigVolumeMountPath}, Ports: []corev1.ContainerPort{ {Name: "internode", ContainerPort: 7000}, + {Name: "internode-tls", ContainerPort: 7001}, {Name: "cql", ContainerPort: 9042}, {Name: "jmx", ContainerPort: 7199}, }, @@ -124,6 +136,7 @@ func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, d }, }, }, + Env: cdc.Spec.CassandraEnv, ReadinessProbe: &corev1.Probe{ Handler: corev1.Handler{ Exec: &corev1.ExecAction{ @@ -139,6 +152,15 @@ func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, d }, } + if userConfigVolume != nil { + container.Args = append(container.Args, UserConfigVolumeMountPath) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: userConfigVolume.Name, MountPath: UserConfigVolumeMountPath}) + } + + if userSecretVolume != nil { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: userSecretVolume.Name, MountPath: UserSecretVolumeMountPath}) + } + if cdc.Spec.PrometheusSupport == true { container.Ports = append(container.Ports, corev1.ContainerPort{Name: "promql", ContainerPort: 9500}) } @@ -146,19 +168,26 @@ func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, d return container } -func newSidecarContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, podInfoVolume *corev1.Volume) *corev1.Container { - return &corev1.Container{ +func newSidecarContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, podInfoVolume *corev1.Volume, backupSecretVolume *corev1.Volume) *corev1.Container { + container := &corev1.Container{ Name: "sidecar", Image: cdc.Spec.SidecarImage, ImagePullPolicy: cdc.Spec.ImagePullPolicy, Ports: []corev1.ContainerPort{ - {Name: "http", ContainerPort: SidecarApiPort}, + {Name: "http", ContainerPort: sidecar.DefaultSidecarClientOptions.Port}, }, + Env: cdc.Spec.SidecarEnv, VolumeMounts: []corev1.VolumeMount{ {Name: dataVolumeClaim.Name, MountPath: DataVolumeMountPath}, {Name: podInfoVolume.Name, MountPath: "/etc/pod-info"}, }, } + + if backupSecretVolume != nil { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: backupSecretVolume.Name, MountPath: BackupSecretVolumeMountPath}) + } + + return container } func newSysctlLimitsContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter) *corev1.Container { @@ -167,7 +196,7 @@ func newSysctlLimitsContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter Image: cdc.Spec.CassandraImage, ImagePullPolicy: cdc.Spec.ImagePullPolicy, SecurityContext: &corev1.SecurityContext{ - Privileged: func() *bool { b := true; return &b }(), + Privileged: &cdc.Spec.PrivilegedSupported, }, Command: []string{"bash", "-xuec"}, Args: []string{ @@ -176,6 +205,37 @@ func newSysctlLimitsContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter } } +func newUserConfigVolume(rctx *reconciliationRequestContext) *corev1.Volume { + if rctx.cdc.Spec.UserConfigMapVolumeSource == nil { + return nil + } + + return &corev1.Volume{ + Name: rctx.cdc.Spec.UserConfigMapVolumeSource.Name, + VolumeSource: corev1.VolumeSource{ConfigMap: rctx.cdc.Spec.UserConfigMapVolumeSource}, + } +} + +func newUserSecretVolume(rctx *reconciliationRequestContext) *corev1.Volume { + if rctx.cdc.Spec.UserSecretVolumeSource == nil { + return nil + } + return &corev1.Volume{ + Name: rctx.cdc.Spec.UserSecretVolumeSource.SecretName, + VolumeSource: corev1.VolumeSource{Secret: rctx.cdc.Spec.UserSecretVolumeSource}, + } +} + +func newBackupSecretVolume(rctx *reconciliationRequestContext) *corev1.Volume { + if rctx.cdc.Spec.BackupSecretVolumeSource == nil { + return nil + } + return &corev1.Volume{ + Name: rctx.cdc.Spec.BackupSecretVolumeSource.SecretName, + VolumeSource: corev1.VolumeSource{Secret: rctx.cdc.Spec.BackupSecretVolumeSource}, + } +} + func newPodInfoVolume() *corev1.Volume { return &corev1.Volume{ Name: "pod-info", diff --git a/pkg/sidecar/client.go b/pkg/sidecar/client.go index c75228e4..7c6a8ded 100644 --- a/pkg/sidecar/client.go +++ b/pkg/sidecar/client.go @@ -43,7 +43,7 @@ type Client struct { type ClientOptions struct { Secure bool HttpMode bool - Port int + Port int32 Timeout time.Duration }