Download archives only if modified and download and extract in single…

… step (#329) * Only download archives if modified
galaxyproject · Oct 11, 2021 · 305bc54 · 305bc54
1 parent edc8b0e
commit 305bc54
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 3 deletions.
diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl
@@ -166,6 +166,20 @@ until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}}
 {{- end }}
 {{- end -}}
 
+{{/*
+Creates shell commands for downloading and extracting archives if modified
+*/}}
+{{- define "galaxy.extract-archive-if-changed-command" -}}
+if [ -f {{ .extractPath }}/{{ base .downloadUrl }}_timestamp ]; then
+  echo "File {{ .downloadUrl }} previously downloaded. Only downloading if changed...";
+  wget -qO- --header="If-Modified-Since: `cat {{ .extractPath }}/{{ base .downloadUrl }}_timestamp`" {{ .downloadUrl }} | tar -xvz || echo File not changed, ignoring....;
+else
+  echo "File not previously downloaded. Downloading and extracting {{ .downloadUrl }}...";
+  wget -qO- {{ .downloadUrl }} | tar -xvz || exit 1;
+fi;
+wget --server-response --spider {{ .downloadUrl }} 2>&1 | grep -i "Last-Modified: " | cut -c18- > {{ .extractPath }}/{{ base .downloadUrl }}_timestamp;
+echo "Completed download and extraction of: {{ .downloadUrl }}"
+{{- end -}}
 
 
 {{/*

diff --git a/galaxy/templates/jobs-init.yaml b/galaxy/templates/jobs-init.yaml
@@ -198,7 +198,7 @@ spec:
 {{ if .Values.initJob.downloadToolConfs.enabled }}
         - name: {{ .Chart.Name }}-init-cloud-repo
           image: alpine:3.7
-          command: ['sh', '-c', 'wget -O startup.tar.gz {{ .Values.initJob.downloadToolConfs.archives.startup }} && tar -xvf startup.tar.gz > /dev/null && echo "Done" > /galaxy/server/config/mutable/init_clone_done_{{.Release.Revision}}']
+          command: ['sh', '-c', '{{- include "galaxy.extract-archive-if-changed-command" (dict "extractPath" .Values.initJob.downloadToolConfs.volume.mountPath "downloadUrl" .Values.initJob.downloadToolConfs.archives.startup) -}} && echo "Done" > /galaxy/server/config/mutable/init_clone_done_{{.Release.Revision}}']
           volumeMounts:
             - name: galaxy-data
               mountPath: {{ .Values.initJob.downloadToolConfs.volume.mountPath }}
@@ -208,14 +208,14 @@ spec:
               subPath: config
         - name: {{ .Chart.Name }}-init-cloud-repo-partial
           image: alpine:3.7
-          command: ['sh', '-c', 'wget -O partial.tar.gz {{ .Values.initJob.downloadToolConfs.archives.running }} && tar -xvf partial.tar.gz > /dev/null']
+          command: ['sh', '-c', '{{- include "galaxy.extract-archive-if-changed-command" (dict "extractPath" .Values.initJob.downloadToolConfs.volume.mountPath "downloadUrl" .Values.initJob.downloadToolConfs.archives.running) -}}']
           volumeMounts:
             - name: galaxy-data
               mountPath: {{ .Values.initJob.downloadToolConfs.volume.mountPath }}
               subPath: {{ .Values.initJob.downloadToolConfs.volume.subPath }}
         - name: {{ .Chart.Name }}-init-cloud-repo-full
           image: alpine:3.7
-          command: ['sh', '-c', 'wget -O contents.tar.gz {{ .Values.initJob.downloadToolConfs.archives.full }} && tar -xvf contents.tar.gz > /dev/null && chown 101:101 {{ .Values.initJob.downloadToolConfs.volume.mountPath }};']
+          command: ['sh', '-c', '{{- include "galaxy.extract-archive-if-changed-command" (dict "extractPath" .Values.initJob.downloadToolConfs.volume.mountPath "downloadUrl" .Values.initJob.downloadToolConfs.archives.full) -}}']
           volumeMounts:
             - name: galaxy-data
               mountPath: {{ .Values.initJob.downloadToolConfs.volume.mountPath }}