diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 6aae1d47..7436a068 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -47,3 +47,18 @@ jobs: charts-token: ${{ secrets.CHARTS_TOKEN }} github-labels: ${{ github.event.inputs.version-bump }} git-branch: ${{ github.event.inputs.branch-name }} + tag-and-release: + needs: [ package-from-pr, package-from-manual ] + name: Create a tag and GitHub release for this version. + runs-on: ubuntu-latest + if: | + always() + && contains(needs.*.result, 'success') + && !contains(needs.*.result, 'failure') + steps: + - name: Tag and release + run: | + version=v$(cat galaxy/Chart.yaml | grep ^version: | awk '{print $2}') + git tag -a $version -m "Automatic release of $version" + git push origin $version + gh release create $version --generate-notes --latest diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e9ab20ae..bbde0299 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,6 +5,7 @@ on: - master - anvil pull_request: {} + workflow_dispatch: {} jobs: linting: runs-on: ubuntu-latest @@ -33,7 +34,7 @@ jobs: - name: Start k8s locally uses: jupyterhub/action-k3s-helm@v3 with: - k3s-version: v1.26.1+k3s1 # releases: https://github.com/k3s-io/k3s/tags + k3s-version: v1.25.15+k3s2 # releases: https://github.com/k3s-io/k3s/tags metrics-enabled: false traefik-enabled: false - name: Verify function of k8s, kubectl, and helm diff --git a/README.md b/README.md index 33f1ee07..f7684f85 100644 --- a/README.md +++ b/README.md @@ -34,18 +34,40 @@ helm upgrade --install ingress-nginx ingress-nginx \ This chart relies on the features of other charts for common functionality: - [postgres-operator](https://github.com/zalando/postgres-operator) for the database; -- [CSI-S3 chart](https://github.com/ctrox/csi-s3/pull/75/) for linking the - reference data to Galaxy and jobs based on S3FS. +- [galaxy-cvmfs-csi](https://github.com/CloudVE/galaxy-cvmfs-csi-helm) for linking the + reference data to Galaxy and jobs based on CVMFS (default). +- [csi-s3](https://github.com/ctrox/csi-s3/pull/75/) for linking + reference data to Galaxy and jobs based on S3FS (optional/alternative to CVMFS). +- [rabbitmq-cluster-operator](https://github.com/rabbitmq/cluster-operator) for deploying + the message queue. In a production setting, especially if the intention is to run multiple Galaxies in a single cluster, we recommend installing the dependency charts separately once per cluster, and installing Galaxy with `--set postgresql.deploy=false ---set s3csi.deploy=false`. +--set s3csi.deploy=false --set cvmfs.deploy=false --set rabbitmq.deploy=false`. --- ## Installing the chart +### Using the chart from the packaged chart repo + +1. The chart is automatically packaged, versioned and uploaded to a helm repository +on each accepted PR. Therefore, the latest version of the chart can be downloaded +from this repository. + +```console +helm repo add cloudve https://raw.githubusercontent.com/CloudVE/helm-charts/master/ +helm repo update +``` + +2. Install the chart with the release name `my-galaxy`. It is not advisable to + install Galaxy in the `default` namespace. + +```console +helm install my-galaxy-release cloudve/galaxy +``` + ### Using the chart from GitHub repo 1. Clone this repository and add required dependency charts: @@ -68,32 +90,50 @@ In several minute, Galaxy will be available at `/galaxy/` URL of your Kubernetes cluster. If you are running the development Kubernetes, Galaxy will be available at `http://localhost/galaxy/` (note the trailing slash). -### Using the chart from the packaged chart repo +## Uninstalling the chart -1. Instead of using the source code repo, you can install the packaged version -of the chart and hence not need to clone this GitHub repo. The packaged version -may contain a bit older but possibly more stable code than what is the GitHub -repo at any a given point in time. +To uninstall/delete the `my-galaxy` deployment, run: ```console -helm repo add cloudve https://raw.githubusercontent.com/CloudVE/helm-charts/master/ -helm repo update +helm delete my-galaxy ``` -2. Install the chart with the release name `my-galaxy`. It is not advisable to - install Galaxy in the `default` namespace. +if you see that some RabbitMQ and Postgres elements remain after some 10 minutes or more, you should be able to issue: -```console -helm install my-galaxy-release cloudve/galaxy +``` +kubectl delete RabbitmqCluster/my-galaxy-rabbitmq-server +kubectl delete statefulset/galaxy-my-galaxy-postgres ``` -## Uninstalling the chart +it might be needed to remove the finalizer from the RabbitmqCluster if the above doesn't seem to get rid of RabbitmqCluster, through a -To uninstall/delete the `my-galaxy` deployment, run: +``` +kubectl edit RabbitmqCluster/my-galaxy-rabbitmq-server +``` + +remove the finalizer in: -```console -helm delete my-galaxy ``` +apiVersion: rabbitmq.com/v1beta1 +kind: RabbitmqCluster +metadata: + annotations: + meta.helm.sh/release-name: my-galaxy + meta.helm.sh/release-namespace: default + creationTimestamp: "2022-12-19T16:54:33Z" + deletionGracePeriodSeconds: 0 + deletionTimestamp: "2022-12-19T17:41:40Z" + finalizers: + - deletion.finalizers.rabbitmqclusters.rabbitmq.com +``` + +and remove the postgres secret: + +``` +kubectl delete secrets/standby.galaxy-my-galaxy-postgres.credentials.postgresql.acid.zalan.do +``` + +Consider as well that if you set persistence to be enabled, Postgres and Galaxy will leave their PVCs behind, which you might want to delete or not depending on your use case. ## Configuration diff --git a/VALUES.md b/VALUES.md new file mode 100644 index 00000000..bbae4835 --- /dev/null +++ b/VALUES.md @@ -0,0 +1,50 @@ +| Key | Description | +|-----|-------------| +| nameOverride | Partial override of the `galaxy.fullname`. The `.Release.Name` will be prepended to generate the fullname. | +| fullnameOverride | Fully override the `galaxy.fullname` | +| image.repository | Repository containing the Galaxy image. | +| image.tag | Galaxy Docker image tag (generally corresponds to the desired Galaxy version) | +| image.pullPolicy | Galaxy image [pull policy](https://kubernetes.io/docs/concepts/configuration/overview/#container-images) | +| imagePullSecrets | Secrets used to [access a Galaxy image from a private repository](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) | +| trainingHook.enabled | Enable the GTN webhook to link references to tools in tutorials to the corresponding tool panel in Galaxy. | +| trainingHook.url | The training material server used to service the training-material webhook. | +| service.type | The Galaxy service type | +| service.port | The port Galaxy is listening to | +| service.nodePort | The external port exposed on each node | +| metrics.enabled | Enable the metrics server. Defaults to `false` | +| serviceAccount.create | Specifies whether a service account should be created | +| serviceAccount.annotations | Annotations to add to the service account | +| serviceAccount.name | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| rbac.enabled | Does the cluster use role based access control. | +| securityContext.fsGroup | Security context and file system group used by jobs. | +| persistence | Configure the PVC used by Galaxy for local storage. | +| persistence.enabled | Persistence is enabled by default | +| persistence.name | Name of the PVC to create | +| persistence.storageClass | StorageClass for the PVC. Must support `ReadWriteMany`. | +| persistence.existingClaim | The name of an existing PVC to use for persistence. | +| setupJob | tasks to perform once after installation | +| setupJob.createDatabase | create the database | +| setupJob.securityContext.runAsUser | the setup jobs will run as this user | +| setupJob.securityContext.runAsGroup | the `runAsUser` will belong to this group. | +| setupJob.securityContext.fsGroup | the filesystem group | +| setupJob.downloadToolConfs.archives.startup | A tar.gz publicly accessible archive containing AT LEAST conf files and XML tool wrappers. Meant to be enough for Galaxy handlers to startup | +| setupJob.downloadToolConfs.archives.running | A tar.gz publicly accessible archive containing AT LEAST confs, tool wrappers, and scripts excluding test data. Meant to be enough for Galaxy handlers to run jobs. | +| setupJob.downloadToolConfs.archives.full | A tar.gz publicly accessible archive containing the full `tools` directory, including each tool's test data. Meant to be enough to run automated tool-tests, fully mimicking CVMFS setup | +| extraInitContainers | Allow users to specify extra init containers | +| ingress.enabled | Should ingress be enabled. Defaults to `true` | +| ingress.ingressClassName | | +| resources.requests | We recommend updating these based on the usage levels of the server. | +| postgresql.deploy | Whether to deploy the postgresl operator. In general, we recommend installing the operator globally in production. | +| postgresql.existingDatabase | hostname and port of an existing database to use. | +| refdata | Configuration block for reference data | +| refdata.enabled | Whether or not to mount cloud-hosted Galaxy reference data and tools. | +| refdata.type | `s3fs` or `cvmfs`, determines the CSI to use for mounting reference data. `cvmfs` is the default and recommended for the time being. | +| cvmfs | Configuration block if `cvmfs` is used as `refdata.type` | +| cvmfs.deploy | Deploy the Galaxy-CVMFS-CSI Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource | +| s3csi | Configuration block if `s3csi` is used as the `refdata.type` | +| s3csi.deploy | Deploy the CSI-S3 Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource. | +| useSecretConfigs | When this flag is set to true, all configs will be set in secrets, when it is set to false, all configs will be set in configmaps | +| configs | All config files will be relative to `/galaxy/server/config/` directory | +| configs.galaxy\.yml | Galaxy configuration. See the [Galaxy documentation](https://docs.galaxyproject.org/en/master/admin/config.html) for more information. | +| jobs | Additional dynamic rules to map into the container. | +| jobs.priorityClass.enabled | Assign a [priorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) to the dispatched jobs. | diff --git a/galaxy/Chart.yaml b/galaxy/Chart.yaml index fab8fb19..bf97d0f1 100644 --- a/galaxy/Chart.yaml +++ b/galaxy/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: galaxy type: application -version: 5.7.6-anvil.1 -appVersion: "23.1" +version: 5.9.0-anvil.0 +appVersion: "24.0" description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png dependencies: diff --git a/galaxy/templates/_helpers.tpl b/galaxy/templates/_helpers.tpl index 81393c64..7e0df8f6 100644 --- a/galaxy/templates/_helpers.tpl +++ b/galaxy/templates/_helpers.tpl @@ -157,7 +157,7 @@ cp -anL /galaxy/server/config/sanitize_allowlist.txt /galaxy/server/config/mutab cp -anL /galaxy/server/config/data_manager_conf.xml.sample /galaxy/server/config/mutable/shed_data_manager_conf.xml; cp -anL /galaxy/server/config/tool_data_table_conf.xml.sample /galaxy/server/config/mutable/shed_tool_data_table_conf.xml; cp -aruL /galaxy/server/tool-data {{.Values.persistence.mountPath}}/; -cp -aruL /galaxy/server/tools {{.Values.persistence.mountPath}}/tools | true; +cp -aruL /galaxy/server/tools {{.Values.persistence.mountPath}}/; echo "Done" > /galaxy/server/config/mutable/init_mounts_done_{{.Release.Revision}}; {{- end -}} diff --git a/galaxy/templates/configmap-nginx.yaml b/galaxy/templates/configmap-nginx.yaml index 98078d12..d1166b32 100644 --- a/galaxy/templates/configmap-nginx.yaml +++ b/galaxy/templates/configmap-nginx.yaml @@ -78,7 +78,11 @@ data: add_header X-Frame-Options SAMEORIGIN; add_header X-Content-Type-Options nosniff; } - +{{- if .Values.trainingHook.enabled }} + location /training-material/ { + proxy_pass {{ .Values.trainingHook.url }}; + } +{{- end }} # end server } diff --git a/galaxy/templates/configs-galaxy.yaml b/galaxy/templates/configs-galaxy.yaml index bd61eaba..f1b0e229 100644 --- a/galaxy/templates/configs-galaxy.yaml +++ b/galaxy/templates/configs-galaxy.yaml @@ -13,10 +13,11 @@ data: {{- end }} {{- range $key, $entry := .Values.configs -}} {{- if $entry -}} - {{- $key | nindent 2 }}: | - {{- $original := (toYaml $entry) -}} - {{- $nomultiline := (regexReplaceAll "^^\\s*\\|\\n*" $original "") -}} - {{- $nowhitespace := (regexReplaceAll "{{\\s*([^}\\s]+)\\s*}}" $nomultiline "{{$1}}") -}} + {{- $key | nindent 2 -}}: | + {{- /* Don't call toYaml on string objects, only on maps or other complex objects. */}} + {{- $isStringObject := (kindIs "string" $entry) -}} + {{- $original := (ternary $entry (toYaml $entry) $isStringObject) -}} + {{- $nowhitespace := (regexReplaceAll "{{\\s*([^}\\s]+)\\s*}}" $original "{{$1}}") -}} {{- tpl (tpl $nowhitespace $) $ | nindent 4 -}} {{- end -}} {{- end -}} diff --git a/galaxy/templates/hook-cvmfs-fix.yaml b/galaxy/templates/hook-cvmfs-fix.yaml index 12ab3cbd..1caa21ba 100644 --- a/galaxy/templates/hook-cvmfs-fix.yaml +++ b/galaxy/templates/hook-cvmfs-fix.yaml @@ -1,3 +1,5 @@ +{{- if and .Values.refdata.enabled (eq .Values.refdata.type "cvmfs") }} + # Include the code you want to run when both conditions are met apiVersion: batch/v1 kind: Job metadata: @@ -33,3 +35,5 @@ spec: - name: kubectl-script configMap: name: "{{ .Release.Name }}-configmap-cvmfs-fix" +{{- end }} + diff --git a/galaxy/templates/priorityclass-job.yaml b/galaxy/templates/priorityclass-job.yaml index 1d5e2b24..df518387 100644 --- a/galaxy/templates/priorityclass-job.yaml +++ b/galaxy/templates/priorityclass-job.yaml @@ -5,8 +5,8 @@ metadata: name: {{ include "galaxy.pod-priority-class" . }} labels: {{- include "galaxy.labels" . | nindent 4 }} -value: -1000 +value: {{ default "-1000" .Values.jobs.priorityClass.value }} preemptionPolicy: Never globalDefault: false -description: "Galaxy jobs will run when cluster resources are available and will not preempt existing workloads." +description: "By default, Galaxy jobs will not preempt existing workloads but will run when cluster resources are available. Depending on the cluster setup, the preemption value can be modified to prompt scaling of the cluster for the computation." {{ end }} diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 87acf679..09d50603 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -1,19 +1,34 @@ # Default values for Galaxy. # Declare variables to be passed into your templates. +#- Partial override of the `galaxy.fullname`. The `.Release.Name` will be prepended to generate the fullname. nameOverride: "" +#- Fully override the `galaxy.fullname` fullnameOverride: "" image: - repository: quay.io/galaxyproject/galaxy-min - tag: "23.1" # Value must be quoted + #- Repository containing the Galaxy image. + repository: quay.io/galaxyproject/galaxy-anvil + #- Galaxy Docker image tag (generally corresponds to the desired Galaxy version) + tag: "24.0" # Value must be quoted + #- Galaxy image [pull policy](https://kubernetes.io/docs/concepts/configuration/overview/#container-images) pullPolicy: IfNotPresent +#- Secrets used to [access a Galaxy image from a private repository](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) imagePullSecrets: [] +trainingHook: + #- Enable the GTN webhook to link references to tools in tutorials to the corresponding tool panel in Galaxy. + enabled: false + #- The training material server used to service the training-material webhook. + url: https://training.galaxyproject.org/training-material/ + service: + #- The Galaxy service type type: ClusterIP + #- The port Galaxy is listening to port: 8000 + #- The external port exposed on each node nodePort: 30700 workflowHandlers: @@ -141,6 +156,7 @@ celeryBeat: timeoutSeconds: 10 metrics: + #- Enable the metrics server. Defaults to `false` enabled: false annotations: {} podAnnotations: {} @@ -151,25 +167,32 @@ metrics: pullPolicy: IfNotPresent serviceAccount: - # Specifies whether a service account should be created + #- Specifies whether a service account should be created create: true - # Annotations to add to the service account + #- Annotations to add to the service account annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + #- The name of the service account to use. + #- If not set and create is true, a name is generated using the fullname template name: "" rbac: + #- Does the cluster use role based access control. enabled: true securityContext: + #- Security context and file system group used by jobs. fsGroup: 101 +#- Configure the PVC used by Galaxy for local storage. persistence: + #- Persistence is enabled by default enabled: true + #- Name of the PVC to create name: galaxy-pvc annotations: {} + #- StorageClass for the PVC. Must support `ReadWriteMany`. storageClass: "" + #- The name of an existing PVC to use for persistence. existingClaim: null accessMode: ReadWriteMany size: 5Gi @@ -184,11 +207,16 @@ extraVolumeMounts: [] # - name: shared-data # mountPath: /mnt/project/shared-data +#- tasks to perform once after installation setupJob: + #- create the database createDatabase: true securityContext: + #- the setup jobs will run as this user runAsUser: 101 + #- the `runAsUser` will belong to this group. runAsGroup: 101 + #- the filesystem group fsGroup: 101 ttlSecondsAfterFinished: 300 downloadToolConfs: @@ -198,18 +226,19 @@ setupJob: subPath: cvmfsclone # on galaxy-data volume (avoid config, tools, lib, etc...) mountPath: /cvmfs/cloud.galaxyproject.org archives: - # A tar.gz publicly accessible archive containing AT LEAST conf files and XML tool wrappers. - # Meant to be enough for Galaxy handlers to startup + #- A tar.gz publicly accessible archive containing AT LEAST conf files and XML tool wrappers. + #- Meant to be enough for Galaxy handlers to startup startup: https://storage.googleapis.com/cloud-cvmfs/startup.tar.gz - # A tar.gz publicly accessible archive containing AT LEAST confs, tool wrappers, and scripts - # excluding test data. - # Meant to be enough for Galaxy handlers to run jobs. + #- A tar.gz publicly accessible archive containing AT LEAST confs, tool wrappers, and scripts + #- excluding test data. + #- Meant to be enough for Galaxy handlers to run jobs. running: https://storage.googleapis.com/cloud-cvmfs/partial.tar.gz - # A tar.gz publicly accessible archive containing the full `tools` directory, - # including each tool's test data. - # Meant to be enough to run automated tool-tests, fully mimicking CVMFS setup + #- A tar.gz publicly accessible archive containing the full `tools` directory, + #- including each tool's test data. + #- Meant to be enough to run automated tool-tests, fully mimicking CVMFS setup full: https://storage.googleapis.com/cloud-cvmfs/contents.tar.gz +#- Allow users to specify extra init containers extraInitContainers: [] # - name: my-first-container # applyToJob: true @@ -239,7 +268,9 @@ extraEnv: [] # value: MY_VALUE ingress: + #- Should ingress be enabled. Defaults to `true` enabled: true + #- ingressClassName: nginx canary: enabled: true @@ -253,13 +284,14 @@ ingress: - host: ~ paths: - path: "/galaxy" + - path: "/training-material" tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local resources: - # We recommend updating these based on the usage levels of the server + #- We recommend updating these based on the usage levels of the server. requests: cpu: 100m memory: 1G @@ -278,8 +310,10 @@ affinity: {} postgresql: # We are not using the Operator with AnVIL so do not enable it enabled: false + # Whether to deploy the postgresl operator. + # In general, we recommend installing the operator globally in production. deploy: true - # hostname and port of an existing database to use. + #- hostname and port of an existing database to use. existingDatabase: galaxyDatabaseUser: galaxydbuser galaxyConnectionParams: "" @@ -306,17 +340,25 @@ postgresql: name: '{{default (printf "%s-galaxy-secrets" .Release.Name) .Values.galaxyExistingSecret}}' key: '{{default "galaxy-db-password" .Values.galaxyExistingSecretKeyRef}}' +#- Configuration block for reference data refdata: + #- Whether or not to mount cloud-hosted Galaxy reference data and tools. enabled: true + #- `s3fs` or `cvmfs`, determines the CSI to use for mounting reference data. + #-`cvmfs` is the default and recommended for the time being. type: cvmfs pvc: size: 10Gi +#- Configuration block if `cvmfs` is used as `refdata.type` cvmfs: + #- Deploy the Galaxy-CVMFS-CSI Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource deploy: true storageClassName: "{{ .Release.Name }}-cvmfs" +#- Configuration block if `s3csi` is used as the `refdata.type` s3csi: + #- Deploy the CSI-S3 Helm Chart. This is an optional dependency, and for production scenarios it should be deployed separately as a cluster-wide resource. deploy: false images: csi: cloudve/csi-s3:0.31.3 @@ -335,11 +377,11 @@ s3csi: usePrefix: true prefix: /galaxy/v1/data.galaxyproject.org -# When this flag is set to true, all configs will be set in secrets, -# when it is set to false, all configs will be set in configmaps +#- When this flag is set to true, all configs will be set in secrets, +#- when it is set to false, all configs will be set in configmaps useSecretConfigs: false -# All files will be relative to `/galaxy/server/config/` directory +#- All config files will be relative to `/galaxy/server/config/` directory configs: job_conf.yml: runners: @@ -388,13 +430,16 @@ configs: rules_module: tpv.rules docker_default_container_id: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" tpv_config_files: - - https://raw.githubusercontent.com/galaxyproject/tpv-shared-database/main/tools.yml + - https://gxy.io/tpv/db.yml - lib/galaxy/jobs/rules/tpv_rules_local.yml # limits: # - type: registered_user_concurrent_jobs # value: 5 # - type: anonymous_user_concurrent_jobs # value: 2 + + #- Galaxy configuration. See the [Galaxy documentation](https://docs.galaxyproject.org/en/master/admin/config.html) + #- for more information. galaxy.yml: galaxy: galaxy_url_prefix: "{{ .Values.ingress.path }}" @@ -411,7 +456,7 @@ configs: sanitize_allowlist_file: "/galaxy/server/config/mutable/sanitize_allowlist.txt" tool_config_file: "/galaxy/server/config/tool_conf.xml{{if .Values.setupJob.downloadToolConfs.enabled}},{{ .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_tool_conf.xml{{end}}" shed_tool_config_file: "/galaxy/server/config/mutable/editable_shed_tool_conf.xml" - enable_tool_document_cache: true + enable_tool_document_cache: false tool_data_table_config_path: |- {{ if .Values.setupJob.downloadToolConfs.enabled }} {{- .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_tool_data_table_conf.xml @@ -499,11 +544,13 @@ configs: tool_conf.xml: | {{- (.Files.Get "files/configs/tool_conf.xml") }} -# Additional dynamic rules to map into the container. +#- Additional dynamic rules to map into the container. jobs: priorityClass: + #- Assign a [priorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) to the dispatched jobs. enabled: true existingClass: "" + value: -1000 rules: tpv_rules_local.yml: global: @@ -578,7 +625,7 @@ extraFileMappings: - +
@@ -642,7 +689,7 @@ tusd: fsGroup: 101 image: repository: tusproject/tusd - tag: v1.9.1 + tag: v1.13.0 pullPolicy: IfNotPresent ingress: enabled: true diff --git a/scripts/tag-and-release.sh b/scripts/tag-and-release.sh new file mode 100755 index 00000000..0a110ada --- /dev/null +++ b/scripts/tag-and-release.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# Determine our name for use in the help message. +SCRIPT=$(basename $(realpath $0)) + +# ANSI color codes for the console. +reset="\033[0m" +bold="\033[1m" +ital="\033[3m" # does not work on OS X + +# Function used to highlight text. +function hi() { + echo -e "$bold$@$reset" +} + +# The last version that was tagged. +LATEST=$(git describe --tags `git rev-list --tags --max-count=1` | tr -d 'v') + +# Date since the last release. We will search for commits newer than this. +SINCE=$(git log -1 --format=%as v$LATEST) + +function help() { + less -RX << EOF + +$(hi NAME) + $SCRIPT + +$(hi DESCRIPTION) + Tags and generates a GitHub release for all commits that have "Automatic" in the + commit messages. These will be the commits that were generated by the packaging + GitHub action. + +$(hi SYNOPSIS) + $SCRIPT --since DATE --latest TAG_NAME + +$(hi OPTIONS) + -l|--latest the tag name of the last commit that was tagged. Default to $LATEST + -s|--since search the git log after this date (YYYY-MM-DD). Defaults to $SINCE + -h|--help prints this help message and exits + + The $(hi --latest) and $(hi --since) fields will be determined from the Git log + if they are not provided. + +Press $(hi Q) to quit this help. + +EOF +} + +while [[ $# > 0 ]] ; do + case $1 in + -l|--latest) + LATEST=$2 + shift + ;; + -s|--since) + SINCE=$2 + shift + ;; + -h|--help|help) + help + exit + ;; + *) + echo "Invalid option $1" + echo "Run $(hi $SCRIPT help) for usage information" + exit + esac + shift +done +PREVIOUS="v$LATEST" + +# Search the git log for commits that did an Automatic version bump. +git log --oneline --grep=Automatic --since=$SINCE | awk '{print $1,$NF}' | grep -v $LATEST | tail -r | while read -r line ; do + commit=$(echo $line | awk '{print $1}') + tag=v$(echo $line | awk '{print $2}') + # Get the actual date the commit was made + export GIT_COMMITTER_DATE=$(git show --format=%aD $commit | head -1) + # Tag the commit + echo "Tagging $commit as $tag $GIT_COMMITTER_DATE" + git checkout $commit + git tag -a -m "Automatic tagging of $tag" $tag + git push origin $tag + # Generate the release. + gh release create $tag --generate-notes --latest --notes-start-tag $PREVIOUS + PREVIOUS=$tag +done +git checkout master +echo "Done." \ No newline at end of file diff --git a/scripts/values_table.py b/scripts/values_table.py index e3f9903f..2da51487 100644 --- a/scripts/values_table.py +++ b/scripts/values_table.py @@ -49,6 +49,8 @@ "jobs.", "refdata.deploy", "refdata.enabled", + "cvmfs", + "s3csi", "refdata.", "setupJob.", "ingress.", @@ -75,10 +77,12 @@ "postgresql.", "s3csi.", "tusd.", + "rabbitmq", + "trainingHook", ] # Entries that should be ignored. -ignored = [ ] +ignored = [ 'startupProbe', 'readinesProbe', 'livenessProbe' ] longest_key = -1 longest_desc = -1 @@ -120,6 +124,8 @@ def print_table(): numbered_order[full_key] = i for key in sorted(set(key_list), key=lambda d: numbered_order[d]): print_table_row(key) + # for key in README_ORDER: + # print_table_row(key) # Prints a single row in the table. def print_table_row(key): @@ -156,7 +162,7 @@ def parse_value(key, value, label=None): record_key('postgresql.enabled') return - if label in ignored: + if label in ignored or key in ignored: # print(f"Ignoring {label}") return diff --git a/scripts/yamldoc.py b/scripts/yamldoc.py new file mode 100644 index 00000000..99cb9890 --- /dev/null +++ b/scripts/yamldoc.py @@ -0,0 +1,121 @@ +# This is NOT a YAML parser. This script does the bare minimum to match +# comments with keys but may get it wrong. Output from this script MUST be +# curated by a human. + +import sys +import argparse + +# Character string used to indicate the comment is a YamlDoc comment. +chars = '#-' + + +class LineProvider: + ''' + A simple iterator type class. Mostly needed because occasionally we need + to push a line back after searching for the end of a multi-line string. + Also collapses lines that end with '\' into a single line. + ''' + def __init__(self, inputPath: str): + with open(inputPath) as f: + self._lines = f.read().splitlines() + self._current = 0 + self._previous = None + + def hasNext(self): + return self._current < len(self._lines) + + + def getNext(self): + self._previous = self._current + line = self._collect_line() + return line + + def back(self): + if self._previous is not None: + self._current = self._previous + else: + self._current -= 1 + + def _next_line(self): + line = self._lines[self._current].rstrip(' ') + self._current += 1 + return line + + def _collect_line(self): + segments = [] + collecting = True + while collecting and self.hasNext(): + line = self._next_line() + if line.endswith('\\'): + line = line.rstrip('\\') + else: + collecting = False + segments.append(line) + return ' '.join(segments) + + + +def get_line_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def print_table_row(path: list, key: str, comments: list): + key = key.replace('.', '\.') + if len(path) == 0: + full_path = key + else: + full_path = f"{'.'.join(path)}.{key}" + print(f"| {full_path} | { ' '.join(comments)} |") + + +def main(filename: str, chars: str): + lp = LineProvider(filename) + path = [] + doc_lines = [] + indent_stack = [ 0 ] + key = '' + print("| Key | Description |") + print("|-----|-------------|") + while lp.hasNext(): + line = lp.getNext().rstrip() + if line.lstrip().startswith(chars): + doc_lines.append(line.replace(chars, '').strip()) + continue + + if len(line.strip()) == 0 or line.lstrip().startswith('#'): + continue + indent = get_line_indent(line) + if indent > indent_stack[-1]: + indent_stack.append(indent) + path.append(key) + elif indent < indent_stack[-1]: + while indent != indent_stack[-1]: + indent_stack.pop() + path.pop() + + if ':' in line: + kv = line.split(':') + key = kv[0].strip() + if len(doc_lines) > 0: + print_table_row(path, key, doc_lines) + doc_lines = [] + # Skip over multi-line strings + if line.endswith('|') or line.endswith('|-') or line.endswith('>-'): + while lp.hasNext() and get_line_indent(line) >= indent: + line = lp.getNext() + # The above loop finds the first line that is NOT part of the + # multi-line string so we need to put that line back into the + # buffer. + lp.back() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="yamldoc", + description="Generate a Markdown table with descriptions of the YAML keys in a Helm Chart's vales.yaml file.", + epilog="Copyright 2023 The Galaxy Project" + ) + parser.add_argument('filename', nargs=1) + parser.add_argument('-c', '--chars', default="#-") + args = parser.parse_args() + main(args.filename[0], args.chars) \ No newline at end of file