Merge pull request #396 from galaxyproject/add_back_support_for_cvmfs

Add support for cvmfs-csi v2
galaxyproject · Jan 9, 2023 · 53f3ed7 · 53f3ed7
2 parents 5fca28d + 14db17e
commit 53f3ed7
Show file tree

Hide file tree

Showing 12 changed files with 91 additions and 84 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -47,7 +47,7 @@ jobs:
       - name: Helm repo add galaxy
         run: helm repo add galaxy https://github.com/CloudVE/helm-charts/raw/master
       - name: Helm install Galaxy
-        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set resources.requests.memory=0Mi,resources.requests.cpu=0m --wait --timeout=720s'
+        run: time bash -c 'helm install --create-namespace -n galaxy galaxy ./galaxy --set persistence.accessMode="ReadWriteOnce" --set resources.requests.memory=0Mi,resources.requests.cpu=0m,cvmfs.cvmfscsi.cache.alien.enabled=false --wait --timeout=720s'
       - name: Get events
         run: kubectl get events -n galaxy; kubectl get events -n csi-drivers;
         if: always()

diff --git a/README.md b/README.md
@@ -126,7 +126,7 @@ current default values can be found in `values.yaml` file.
 | `s3csi.deploy`                             | Deploy the CSI-S3 Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource.          |
 | `cvmfs.deploy`                             | Deploy the Galaxy-CVMFS-CSI Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource                                           |
 | `cvmfs.enabled`                            | Enable use of CVMFS in configs, and deployment of CVMFS Persistent Volume Claims for Galaxy                                                                                                                |
-| `cvmfs.galaxyPersistentVolumeClaims.{}`    | Persistent Volume Claims to deploy for CVMFS repositories. See <a href="galaxy/values.yaml">`values.yaml`</a> for examples.                                                                                                                 |
+| `cvmfs.pvc.{}`                             | Persistent Volume Claim to deploy for CVMFS repositories. See <a href="galaxy/values.yaml">`values.yaml`</a> for examples.                                                                                                                 |
 | `setupJob.ttlSecondsAfterFinished`          | Sets `ttlSecondsAfterFinished` for the initialization jobs. See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/#ttl-controller) for more details.       |
 | `setupJob.downloadToolConfs.enabled`        | Download configuration files and the `tools` directory from an archive via a job at startup                                                                                                                |
 | `setupJob.downloadToolConfs.archives.startup` | A URL to a `tar.gz` publicly accessible archive containing AT LEAST conf files and XML tool wrappers. Meant to be enough for Galaxy handlers to startup.                                                   |

diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml
@@ -20,6 +20,13 @@ dependencies:
     alias: s3csi
     tags:
       - deploy-s3csi
+  - name: galaxy-cvmfs-csi
+    repository: https://raw.githubusercontent.com/cloudve/helm-charts/master/
+    version: 2.0.0
+    condition: cvmfs.deploy
+    alias: cvmfs
+    tags:
+      - deploy-cvmfs
   - name: rabbitmq-cluster-operator
     repository: https://charts.bitnami.com/bitnami
     version: 2.6.12

diff --git a/galaxy/templates/deployment-celery-beat.yaml b/galaxy/templates/deployment-celery-beat.yaml
@@ -136,10 +136,13 @@ spec:
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
             {{- if .Values.refdata.enabled }}
-            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq .Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if .Values.extraVolumeMounts }}
             {{- tpl (.Values.extraVolumeMounts | toYaml | nindent 12) . }}
@@ -178,11 +181,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if .Values.refdata.enabled }}
-        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if .Values.extraVolumes }}
         {{- tpl (.Values.extraVolumes | toYaml | nindent 8) . }}

diff --git a/galaxy/templates/deployment-celery.yaml b/galaxy/templates/deployment-celery.yaml
@@ -136,10 +136,13 @@ spec:
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
             {{- if .Values.refdata.enabled }}
-            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq .Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if .Values.extraVolumeMounts }}
             {{- tpl (.Values.extraVolumeMounts | toYaml | nindent 12) . }}
@@ -178,11 +181,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if .Values.refdata.enabled }}
-        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if .Values.extraVolumes }}
         {{- tpl (.Values.extraVolumes | toYaml | nindent 8) . }}

diff --git a/galaxy/templates/deployment-job.yaml b/galaxy/templates/deployment-job.yaml
@@ -143,10 +143,13 @@ spec:
               subPath: {{ $.Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
             {{- if $.Values.refdata.enabled }}
-            {{- range $key, $entry := $.Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq $.Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if $.Values.extraVolumeMounts }}
             {{- tpl ($.Values.extraVolumeMounts | toYaml | nindent 12) $ }}
@@ -188,11 +191,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if $.Values.refdata.enabled }}
-        {{- range $key, $entry := $.Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if $.Values.extraVolumes }}
         {{- tpl ($.Values.extraVolumes | toYaml | nindent 8) $ }}

diff --git a/galaxy/templates/deployment-web.yaml b/galaxy/templates/deployment-web.yaml
@@ -131,10 +131,13 @@ spec:
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
             {{- if .Values.refdata.enabled }}
-            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq .Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if .Values.extraVolumeMounts }}
             {{- tpl (.Values.extraVolumeMounts | toYaml | nindent 12) . }}
@@ -173,11 +176,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if .Values.refdata.enabled }}
-        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if .Values.extraVolumes }}
         {{- tpl (.Values.extraVolumes | toYaml | nindent 8) . }}

diff --git a/galaxy/templates/deployment-workflow.yaml b/galaxy/templates/deployment-workflow.yaml
@@ -140,10 +140,13 @@ spec:
               subPath: {{ .Values.setupJob.downloadToolConfs.volume.subPath }}
             {{- end }}
             {{- if .Values.refdata.enabled }}
-            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq .Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if .Values.extraVolumeMounts }}
             {{- tpl (.Values.extraVolumeMounts | toYaml | nindent 12) . }}
@@ -185,11 +188,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if .Values.refdata.enabled }}
-        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if .Values.extraVolumes }}
         {{- tpl (.Values.extraVolumes | toYaml | nindent 8) . }}

diff --git a/galaxy/templates/jobs-init.yaml b/galaxy/templates/jobs-init.yaml
@@ -27,7 +27,7 @@ spec:
       initContainers:
         - name: {{ .Chart.Name }}-wait-postgres
           image: alpine:3.16
-          command: ['sh', '-c', 'chown 101:101 {{ .Values.persistence.mountPath }}; until nc -z -w3 {{ template "galaxy-postgresql.fullname" . }} 5432; do echo waiting for galaxy-postgres service; sleep 1; done; echo done;']
+          command: ['sh', '-c', 'echo Chown mount path; chown 101:101 {{ .Values.persistence.mountPath }}; echo Begin waiting for postgres; until nc -z -w3 {{ template "galaxy-postgresql.fullname" . }} 5432; do echo waiting for galaxy-postgres service; sleep 1; done; echo done;']
           volumeMounts:
             - name: galaxy-data
               mountPath: {{ .Values.persistence.mountPath }}
@@ -76,10 +76,13 @@ spec:
               mountPath: /galaxy/server/config/mutable/
               subPath: config
             {{- if .Values.refdata.enabled }}
-            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq .Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if .Values.extraVolumeMounts }}
             {{- tpl (.Values.extraVolumeMounts | toYaml | nindent 12) . }}
@@ -116,11 +119,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if .Values.refdata.enabled }}
-        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if .Values.extraVolumes }}
         {{- tpl (.Values.extraVolumes | toYaml | nindent 8) . }}
@@ -183,10 +184,13 @@ spec:
               mountPath: /galaxy/server/config/mutable/
               subPath: config
             {{- if .Values.refdata.enabled }}
-            {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-            - name: refdata-gxy-{{ $key }}
-              mountPath: {{ $entry.mountPath }}
-            {{- end -}}
+            - name: refdata-gxy
+              mountPath: /cvmfs/data.galaxyproject.org
+              subPath: data.galaxyproject.org
+            {{- if eq .Values.refdata.type "cvmfs" }}
+              # CVMFS automount volumes must be mounted with HostToContainer mount propagation.
+              mountPropagation: HostToContainer
+            {{- end }}
             {{- end }}
             {{- if .Values.extraVolumeMounts }}
             {{- tpl (.Values.extraVolumeMounts | toYaml | nindent 12) . }}
@@ -249,11 +253,9 @@ spec:
           emptyDir: {}
           {{- end }}
         {{- if .Values.refdata.enabled }}
-        {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims }}
-        - name: refdata-gxy-{{ $key }}
+        - name: refdata-gxy
           persistentVolumeClaim:
-            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc
-        {{- end }}
+            claimName: {{ template "galaxy.fullname" $ -}}-refdata-gxy-pvc
         {{- end }}
         {{- if .Values.extraVolumes }}
         {{- tpl (.Values.extraVolumes | toYaml | nindent 8) . }}

diff --git a/galaxy/templates/pvcs-cvmfs.yaml → galaxy/templates/pvc-refdata.yaml b/galaxy/templates/pvcs-cvmfs.yaml → galaxy/templates/pvc-refdata.yaml
@@ -1,21 +1,21 @@
 {{- if .Values.refdata.enabled }}
-{{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims -}}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: {{ include "galaxy.fullname" $ }}-refdata-gxy-{{ $key }}-pvc
+  name: {{ include "galaxy.fullname" $ }}-refdata-gxy-pvc
   labels:
     {{- include "galaxy.labels" $ | nindent 4 }}
 spec:
   accessModes:
   - ReadOnlyMany
   resources:
     requests:
-      storage: {{ $entry.storage }}
+      storage: {{ .Values.refdata.pvc.size }}
 {{- if eq $.Values.refdata.type "s3csi" }}
   volumeName: {{ include "galaxy.fullname" $ }}-refdata-gxy-pv
 {{- end }}
-  storageClassName: {{ $entry.storageClassName }}
----
+{{- if eq $.Values.refdata.type "cvmfs" }}
+  storageClassName: {{ $.Release.Name }}-cvmfs
 {{- end }}
+---
 {{- end }}
diff --git a/galaxy/values.yaml b/galaxy/values.yaml
@@ -302,19 +302,15 @@ postgresql:
 
 refdata:
   enabled: true
-  type: s3csi
-  galaxyPersistentVolumeClaims:
-    data:
-      storage: 10Gi
-      storageClassName: refdata-gxy-data
-      mountPath: /cvmfs/data.galaxyproject.org
-    # cloud:
-    #   storage: 1Gi
-    #   storageClassName: cvmfs-gxy-cloud
-    #   mountPath: /cvmfs/cloud.galaxyproject.org
+  type: cvmfs
+  pvc:
+    size: 10Gi
 
-s3csi:
+cvmfs:
   deploy: true
+
+s3csi:
+  deploy: false
   images:
     csi: cloudve/csi-s3:0.31.3
   storageClass:
@@ -349,9 +345,7 @@ configs:
         k8s_persistent_volume_claims: |-
           {{ template "galaxy.pvcname" . -}}:{{ .Values.persistence.mountPath -}}
           {{- if .Values.refdata.enabled -}}
-          {{- range $key, $entry := .Values.refdata.galaxyPersistentVolumeClaims -}}
-          ,{{- template "galaxy.fullname" $ -}}-refdata-gxy-{{ $key }}-pvc:{{ $entry.mountPath -}}
-          {{- end -}}
+          ,{{- template "galaxy.fullname" $ -}}-refdata-gxy-pvc/data.galaxyproject.org:/cvmfs/data.galaxyproject.org
           {{- end -}}
           {{- if .Values.setupJob.downloadToolConfs.enabled -}}
           ,{{ template "galaxy.pvcname" . -}}/{{ .Values.setupJob.downloadToolConfs.volume.subPath }}:{{ .Values.setupJob.downloadToolConfs.volume.mountPath -}}
@@ -420,18 +414,18 @@ configs:
         {{ if .Values.setupJob.downloadToolConfs.enabled }}
         {{- .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_tool_data_table_conf.xml
         {{- if .Values.refdata.enabled -}}
-        ,{{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/tool_data_table_conf.xml,{{.Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/byhand/location/tool_data_table_conf.xml
+        ,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml
         {{- end }}
         {{- else }}
         {{- if .Values.refdata.enabled -}}
-        {{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/tool_data_table_conf.xml,{{.Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/byhand/location/tool_data_table_conf.xml
+        /cvmfs/data.galaxyproject.org//managed/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml
         {{- end }}
         {{- end }}
       tool_dependency_dir: "{{.Values.persistence.mountPath}}/deps"
       job_config_file: "/galaxy/server/config/job_conf.yml"
       builds_file_path: |-
         {{ if .Values.refdata.enabled }}
-        {{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/location/builds.txt
+        /cvmfs/data.galaxyproject.org/managed/location/builds.txt
         {{- end }}
       containers_resolvers_config_file: "/galaxy/server/config/container_resolvers_conf.xml"
       workflow_schedulers_config_file: "/galaxy/server/config/workflow_schedulers_conf.xml"
@@ -446,7 +440,7 @@ configs:
         {{include "galaxy.add_trailing_slash" .Values.ingress.path}}_x_accel_redirect
       len_file_path: |-
         {{ if .Values.refdata.enabled -}}
-        {{- .Values.refdata.galaxyPersistentVolumeClaims.data.mountPath}}/managed/len/ucsc
+        /cvmfs/data.galaxyproject.org/managed/len/ucsc
         {{- end }}
   container_resolvers_conf.xml: |
     <containers_resolvers>
@@ -483,7 +477,7 @@ configs:
     toolshed.g2.bx.psu.edu/repos/iuc/snpeff/snpEff/4.3+T.galaxy1
   build_sites.yml:
     - type: ucsc
-      file: "{{if .Values.refdata.enabled}}{{.Values.refdata.galaxyPersistentVolumeClaims.data.mountPath -}}/managed/location/ucsc_build_sites.txt{{else}}/galaxy/server/tool-data/shared/ucsc/ucsc_build_sites.txt.sample{{end}}"
+      file: "{{if .Values.refdata.enabled}}/cvmfs/data.galaxyproject.org/managed/location/ucsc_build_sites.txt{{else}}/galaxy/server/tool-data/shared/ucsc/ucsc_build_sites.txt.sample{{end}}"
       display: [main,archaea,ucla]
     - type: gbrowse
       file: "/galaxy/server/tool-data/shared/gbrowse/gbrowse_build_sites.txt"

diff --git a/scripts/values_table.py b/scripts/values_table.py
@@ -29,7 +29,6 @@
     'extraInitContainers': 'extraInitContainers.[]',
     'extraEnv': 'extraInitContainers.[]',
     'jobs\\.rules\\..*': 'jobs.rules',
-    'refdata\\.galaxyPersistentVolumeClaims\\..*': 'refdata.galaxyPersistentVolumeClaims.{}',
     'jobHandlers\\..*': 'jobHandlers.{}',
     'webHandlers\\..*': 'webHandlers.{}',
     'workflowHandlers\\..*': 'workflowHandlers.{}',