use non-root C* images (k8ssandra#336)

* Add ability to define s3_compatible settings as well as aws region * Change storage_properties to use map with open properties. Also, simplify the helm chart rendering, remove restriction on the type of storage and remove the restriction of bucketSecret for S3 * Fixes and a test * Fix tests * use non-root C* images * fix wget command, update groupid and userid * generate jvm11-clients.options which is needed for nodetool This is tempoary until datastax/cass-config-definitions#49 is merged. * simply the command * fix unit tests * rebase, fix security contexts, and fix google storage support * fix tests * update default management-api images * fix configmap name and introduce helper template to reduce duplication * refactor common medusa env var code into a helper template * move medusa properties up a level * fix syntax error * remove SecurityContext and update Medusa image The SecurityContext isn't needed since the Cassandra and Medusa images are both already configured to run as the cassandra user/group. * rebase, fix merge conflicts, bump chart version * remove unused code * updates from PR review * update comment on supported C* versions and fix bad merge * add 4.0 logic for jvm options and update tests Co-authored-by: Michael Burman <[email protected]> Co-authored-by: Michael Burman <[email protected]> Co-authored-by: Erik Merkle <[email protected]>
jeffreyscarpenter · Mar 10, 2021 · 7a817b5 · 7a817b5
1 parent 061743f
commit 7a817b5
Show file tree

Hide file tree

Showing 10 changed files with 276 additions and 204 deletions.
diff --git a/charts/k8ssandra/Chart.yaml b/charts/k8ssandra/Chart.yaml
@@ -3,7 +3,7 @@ name: k8ssandra
 description: |
   Provisions and configures an instance of the entire K8ssandra stack. This includes Apache Cassandra, Stargate, Reaper, Medusa, Prometheus, and Grafana.
 type: application
-version: 0.52.0
+version: 0.53.0
 appVersion: 3.11.10
 
 dependencies:
@@ -20,7 +20,7 @@ dependencies:
   - name: medusa-operator
     version: 0.27.0
     repository: file://../medusa-operator
-    condition: backupRestore.medusa.enabled
+    condition: medusa.enabled
 
   - name: k8ssandra-common
     version: 0.28.0

diff --git a/charts/k8ssandra/templates/_helpers.tpl b/charts/k8ssandra/templates/_helpers.tpl
@@ -153,4 +153,39 @@ Set default num_tokens based on the server version
       num_tokens: 16
     {{- end }}
     {{- end }}
-{{- end }}
+{{- end }}
+
+{{- define "medusa.configMapName" -}}
+{{ .Release.Name }}-medusa
+{{- end }}
+
+{{/*
+Creates Cassandra auth environment variables if authentication is enabled.
+*/}}
+{{- define "medusa.cassandraAuthEnvVars" -}}
+{{- if .Values.cassandra.auth.enabled }}
+  {{- if .Values.medusa.cassandraUser.secret }}
+    {{- nindent 10 "- name: CQL_USERNAME" }}
+    {{- nindent 12 "valueFrom:" }}
+    {{- nindent 14 "secretKeyRef:" }}
+    {{- nindent 16 (print "name: " .Values.medusa.cassandraUser.secret) }}
+    {{- nindent 16 "key: username" }}
+    {{- nindent 10 "- name: CQL_PASSWORD" }}
+    {{- nindent 12 "valueFrom:" }}
+    {{- nindent 14 "secretKeyRef:" }}
+    {{- nindent 16 (print "name: " .Values.medusa.cassandraUser.secret) }}
+    {{- nindent 16 "key: password" }}
+  {{- else }}
+    {{- nindent 10 "- name: CQL_USERNAME" -}}
+    {{- nindent 12 "valueFrom:" }}
+    {{- nindent 14 "secretKeyRef:" }}
+    {{- nindent 16 (print "name: " (include "k8ssandra.clusterName" . ) "-medusa") }}
+    {{- nindent 16 "key: username" }}
+    {{- nindent 10 "- name: CQL_PASSWORD" }}
+    {{- nindent 12 "valueFrom:" }}
+    {{- nindent 14 "secretKeyRef:" }}
+    {{- nindent 16 (print "name: " (include "k8ssandra.clusterName" . ) "-medusa") }}
+    {{- nindent 16 "key: password" }}
+  {{- end -}}
+{{- end }}
+{{- end }}
diff --git a/charts/k8ssandra/templates/cassandra/cassdc.yaml b/charts/k8ssandra/templates/cassandra/cassdc.yaml
@@ -1,5 +1,5 @@
 {{- $datacenter := (index .Values.cassandra.datacenters 0) -}}
-{{- $medusaImage := (printf "%s:%s" .Values.backupRestore.medusa.image.repository .Values.backupRestore.medusa.image.tag) -}}
+{{- $medusaImage := (printf "%s:%s" .Values.medusa.image.repository .Values.medusa.image.tag) -}}
 
 {{- if and (not .Values.cassandra.image)  (not (hasKey .Values.cassandra.versionImageMap .Values.cassandra.version)) }}
   {{- fail (print .Values.cassandra.version " is not a supported Cassandra version") }}
@@ -18,7 +18,7 @@ spec:
   clusterName: {{ include "k8ssandra.clusterName" . }}
   serverType: cassandra
   serverVersion: {{ .Values.cassandra.version | quote }}
-  dockerImageRunsAsCassandra: false
+  dockerImageRunsAsCassandra: true
   serverImage: {{ default (get .Values.cassandra.versionImageMap .Values.cassandra.version) .Values.cassandra.image }}
   managementApiAuth:
     insecure: {}
@@ -57,14 +57,14 @@ spec:
 {{- else if .Values.cassandra.auth.superuser.username }}
   superuserSecretName: {{ include "k8ssandra.clusterName" . }}-superuser
 {{- end }}
-  {{- if or .Values.reaper.enabled .Values.backupRestore.medusa.enabled .Values.stargate.enabled }}
+  {{- if or .Values.reaper.enabled .Values.medusa.enabled .Values.stargate.enabled }}
   users:
   {{- if .Values.reaper.enabled }}
     - secretName: {{ default (printf "%s-%s" (include "k8ssandra.clusterName" .) "reaper") .Values.reaper.cassandraUser.secret }}
       superuser: true
   {{- end }}
-  {{- if .Values.backupRestore.medusa.enabled }}
-    - secretName: {{ default (printf "%s-%s" (include "k8ssandra.clusterName" .) "medusa") .Values.backupRestore.medusa.cassandraUser.secret }}
+  {{- if .Values.medusa.enabled }}
+    - secretName: {{ default (printf "%s-%s" (include "k8ssandra.clusterName" .) "medusa") .Values.medusa.cassandraUser.secret }}
       superuser: true
   {{- end }}
   {{- if .Values.stargate.enabled }}
@@ -90,7 +90,11 @@ spec:
       permissions_update_interval_in_ms: {{ .Values.cassandra.auth.cacheUpdateIntervalMillis }}
       credentials_validity_in_ms: {{ .Values.cassandra.auth.cacheValidityPeriodMillis }}
       credentials_update_interval_in_ms: {{ .Values.cassandra.auth.cacheUpdateIntervalMillis }}
+  {{- if (hasPrefix "3" .Values.cassandra.version) }}
     jvm-options:
+  {{- else }}
+    jvm-server-options:
+  {{- end }}
 {{- include "k8ssandra.configureJvmHeap" . }}
       additional-jvm-opts:
 {{- if .Values.cassandra.auth.enabled }}
@@ -99,15 +103,9 @@ spec:
 {{- end }}
   podTemplateSpec:
     spec:
-{{- if or .Values.reaper.enabled .Values.backupRestore.medusa.enabled }}
       initContainers:
-      {{/* The server-config-init container is "built-in" in that it is provided by
-           cass-operator. We specify it here in order to make it run first. It needs
-           to run first when backup/restore is enabled. The configs need to be generated
-           and present for the medusa-restore initContainer.
-        */}}
       - name: server-config-init
-{{- if .Values.reaper.enabled }}
+     {{- if .Values.reaper.enabled }}
       - name: jmx-credentials
         image: busybox
         imagePullPolicy: IfNotPresent
@@ -129,106 +127,90 @@ spec:
         volumeMounts:
           - mountPath: /config
             name: server-config
-{{- end}}
-{{- if .Values.backupRestore.medusa.enabled }}
+      {{- end }}
+      {{- if (hasPrefix "4" .Values.cassandra.version) }}
+      - name: jvm-client-options
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command:
+          - /bin/sh
+        args:
+          - -c
+          - echo "-Djdk.attach.allowAttachSelf=true" > /config/jvm11-clients.options &&
+            echo "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-exports java.base/jdk.internal.ref=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-exports java.base/sun.nio.ch=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-exports java.sql/java.sql=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/java.lang.module=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/jdk.internal.ref=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/jdk.internal.math=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/jdk.internal.module=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED" >> /config/jvm11-clients.options &&
+            echo "" >> /config/jvm11-clients.options &&
+            echo "# The newline in the end of file is intentional" >> /config/jvm11-clients.options &&
+            echo "" >> /config/jvm11-clients.options &&
+            echo "# The newline in the end of file is intentional" > /config/jvm-clients.options &&
+            echo "" >> /config/jmv-clients.options
+        volumeMounts:
+          - mountPath: /config
+            name: server-config
+      {{- end }}
+      {{- if .Values.medusa.enabled }}
       - name: get-jolokia
         image: busybox
         args:
           - /bin/sh
           - -c
-          - wget https://search.maven.org/remotecontent?filepath=org/jolokia/jolokia-jvm/1.6.2/jolokia-jvm-1.6.2-agent.jar && mv jolokia-jvm-1.6.2-agent.jar /config
+          - wget -O  /config/jolokia-jvm-1.6.2-agent.jar https://search.maven.org/remotecontent?filepath=org/jolokia/jolokia-jvm/1.6.2/jolokia-jvm-1.6.2-agent.jar
         volumeMounts:
           - mountPath: /config
             name: server-config
       - name: medusa-restore
         image: {{ $medusaImage }}
-        imagePullPolicy: {{ .Values.backupRestore.medusa.image.pullPolicy }}
+        imagePullPolicy: {{ .Values.medusa.image.pullPolicy }}
         env:
           - name: MEDUSA_MODE
             value: RESTORE
-        {{- if .Values.cassandra.auth.enabled }}
-        {{- if .Values.backupRestore.medusa.cassandraUser.secret }}
-          - name: CQL_USERNAME
-            valueFrom:
-              secretKeyRef:
-                name: {{ .Values.backupRestore.medusa.cassandraUser.secret }}
-                key: username
-          - name: CQL_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: {{ .Values.backupRestore.medusa.cassandraUser.secret }}
-                key: password
-        {{- else }}
-          - name: CQL_USERNAME
-            valueFrom:
-              secretKeyRef:
-                name: {{ include "k8ssandra.clusterName" . }}-medusa
-                key: username
-          - name: CQL_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: {{ include "k8ssandra.clusterName" . }}-medusa
-                key: password
-        {{- end}}
-        {{- end}}
+          {{- include "medusa.cassandraAuthEnvVars" . }}
         volumeMounts:
-          - name: {{ .Release.Name }}-medusa-config-k8ssandra
+          - name: {{ include "medusa.configMapName" . }}
             mountPath: /etc/medusa
           - name: server-config
             mountPath: /etc/cassandra
           - mountPath: /var/lib/cassandra
             name: server-data
-          - name:  {{ .Values.backupRestore.medusa.bucketSecret }}
+          - name:  {{ .Values.medusa.storageSecret }}
             mountPath: /etc/medusa-secrets
-{{- end}}
-{{- end}}
+      {{- end }}
       containers:
       - name: cassandra
-{{- if or .Values.reaper.enabled .Values.backupRestore.medusa.enabled }}
+       {{- if or .Values.reaper.enabled .Values.medusa.enabled }}
         env:
-{{- if .Values.reaper.enabled }}
+          {{- if .Values.reaper.enabled }}
           - name: LOCAL_JMX
             value: "no"
-{{- end}}
-{{- if .Values.backupRestore.medusa.enabled }}
+          {{- end }}
+       {{- if .Values.medusa.enabled }}
           - name: JVM_EXTRA_OPTS
             value: -javaagent:/etc/cassandra/jolokia-jvm-1.6.2-agent.jar=port=7373,host=localhost
         volumeMounts:
           - name: cassandra-config
             mountPath: /etc/cassandra
       - name: medusa
         image: {{ $medusaImage }}
-        imagePullPolicy: {{ .Values.backupRestore.medusa.image.pullPolicy }}
+        imagePullPolicy: {{ .Values.medusa.image.pullPolicy }}
         ports:
           - containerPort: 50051
         env:
           - name: MEDUSA_MODE
             value: GRPC
-        {{- if .Values.cassandra.auth.enabled }}
-        {{- if .Values.backupRestore.medusa.cassandraUser.secret }}
-          - name: CQL_USERNAME
-            valueFrom:
-              secretKeyRef:
-                name: {{ .Values.backupRestore.medusa.cassandraUser.secret }}
-                key: username
-          - name: CQL_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: {{ .Values.backupRestore.medusa.cassandraUser.secret }}
-                key: password
-        {{- else }}
-          - name: CQL_USERNAME
-            valueFrom:
-              secretKeyRef:
-                name: {{ include "k8ssandra.clusterName" . }}-medusa
-                key: username
-          - name: CQL_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: {{ include "k8ssandra.clusterName" . }}-medusa
-                key: password
-        {{- end}}
-        {{- end}}
+          {{- include "medusa.cassandraAuthEnvVars" . }}
         readinessProbe:
           exec:
             command: [ "/bin/grpc_health_probe", "-addr=:50051" ]
@@ -238,25 +220,25 @@ spec:
             command: [ "/bin/grpc_health_probe", "-addr=:50051" ]
           initialDelaySeconds: 10
         volumeMounts:
-          - name: {{ .Release.Name }}-medusa-config-k8ssandra
+          - name: {{ include "medusa.configMapName" . }}
             mountPath: /etc/medusa
           - name: cassandra-config
             mountPath: /etc/cassandra
           - mountPath: /var/lib/cassandra
             name: server-data
           - mountPath: /etc/medusa-secrets
-            name: {{ .Values.backupRestore.medusa.bucketSecret }}
+            name: {{ .Values.medusa.storageSecret }}
       volumes:
-      - name: {{ .Release.Name }}-medusa-config-k8ssandra
+      - name: {{ include "medusa.configMapName" . }}
         configMap:
-          name: {{ .Release.Name }}-medusa-config-k8ssandra
+          name: {{ include "medusa.configMapName" . }}
           items:
             - key: medusa.ini
               path: medusa.ini
       - name: cassandra-config
         emptyDir: {}
-      - name:  {{ .Values.backupRestore.medusa.bucketSecret }}
+      - name:  {{ .Values.medusa.storageSecret }}
         secret:
-          secretName: {{ .Values.backupRestore.medusa.bucketSecret }}
+          secretName: {{ .Values.medusa.storageSecret }}
 {{- end }}
 {{- end }}
diff --git a/charts/k8ssandra/templates/medusa/medusa-config.yaml b/charts/k8ssandra/templates/medusa/medusa-config.yaml
@@ -1,10 +1,10 @@
-{{- $bucketStorageTypes :=  list "s3" "gcs" -}}
+{{- $storageTypes :=  list "s3" "google_storage" "s3_compatible" "local" -}}
 
-{{- if .Values.backupRestore.medusa.enabled }}
+{{- if .Values.medusa.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Release.Name }}-medusa-config-k8ssandra
+  name: {{ include "medusa.configMapName" . }}
   labels: {{ include "k8ssandra.labels" . | indent 4 }}
 data:
   medusa.ini: |-
@@ -19,21 +19,24 @@ data:
     check_running = nodetool version
 
     [storage]
-  {{- if not (or (eq .Values.backupRestore.medusa.storage "s3") (eq .Values.backupRestore.medusa.storage "gcs") (eq .Values.backupRestore.medusa.storage "s3_compatible") (eq .Values.backupRestore.medusa.storage "local")) }}
-    {{ fail "Accepted storage type values are s3, s3_compatible, local and gcs" }}
+  {{- if (not (has .Values.medusa.storage $storageTypes)) }}
+    {{ fail (print "Accepted storage type values are " $storageTypes) }}
   {{- end }}
-    storage_provider = {{ .Values.backupRestore.medusa.storage }}
-  {{- range $key, $value := .Values.backupRestore.medusa.storage_properties }}
+    storage_provider = {{ .Values.medusa.storage }}
+  {{- range $key, $value := .Values.medusa.storage_properties }}
     {{ $key }} = {{ $value }}
   {{- end }}
-  {{- if eq "local" .Values.backupRestore.medusa.storage }}
-    base_path = {{ .Values.backupRestore.medusa.bucketName }}
+  {{- if eq "local" .Values.medusa.storage }}
+    base_path = {{ .Values.medusa.bucketName }}
   {{- else }}
-    bucket_name = {{ .Values.backupRestore.medusa.bucketName }}
+    bucket_name = {{ .Values.medusa.bucketName }}
   {{- end }}
-    # TODO The file name needs to be parameterized. In the current set up it comes from the secret.
+  {{- if (eq .Values.medusa.storage "s3") }}
     key_file = /etc/medusa-secrets/medusa_s3_credentials
-{{- if and .Values.backupRestore.medusa.multiTenant (has .Values.backupRestore.medusa.storage $bucketStorageTypes)}}
+  {{- else if (eq .Values.medusa.storage "google_storage") }}
+    key_file = /etc/medusa-secrets/medusa_gcp_key.json
+  {{- end }}
+{{- if .Values.medusa.multiTenant }}
     prefix = {{ .Values.clusterName }}.{{ .Release.Namespace }}
 {{- end }}
 
@@ -46,4 +49,4 @@ data:
 
     [logging]
     level = DEBUG
-  {{- end }}
+  {{- end }}
diff --git a/charts/k8ssandra/templates/medusa/medusa-user-secret.yaml b/charts/k8ssandra/templates/medusa/medusa-user-secret.yaml
@@ -1,13 +1,13 @@
-{{- if .Values.backupRestore.medusa.enabled }}
-{{- if and .Values.cassandra.auth.enabled (not .Values.backupRestore.medusa.cassandraUser.secret) }}
+{{- if .Values.medusa.enabled }}
+{{- if and .Values.cassandra.auth.enabled (not .Values.medusa.cassandraUser.secret) }}
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ include "k8ssandra.clusterName" . }}-medusa
   labels: {{ include "k8ssandra.labels" . | indent 4 }}
 type: Opaque
 data:
-  username: {{ (default "medusa" .Values.backupRestore.medusa.cassandraUser.username) | b64enc | quote }}
+  username: {{ (default "medusa" .Values.medusa.cassandraUser.username) | b64enc | quote }}
   password: {{ include "k8ssandra-common.password" . }}
 {{- end }}
 {{- end }}