Merge pull request #70 from eliteportal/schematic-228-template-workflow

[SCHEMATIC-228] Add action to commit manifest templates to repo

.github/workflows/generate-current-manifests.yml

@@ -0,0 +1,66 @@
+# This workflow automatically generates all manifests currently specified by the data model
+# Outputs are excel files and json schemas
+# Runs AFTER changes to the data model have been reviewed in a PR and merged into main
+
+name: Store Current Manifests
+
+on:
+  push:
+    branches: main
+    paths: 'EL.data.model.*'
+
+  workflow_dispatch:
+
+jobs:
+  generate:
+    name: generate-and-upload-manifests
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      SCHEMATIC_SERVICE_ACCOUNT_CREDS: ${{ secrets.SCHEMATIC_SERVICE_ACCOUNT_CREDS }}
+      SYNAPSE_AUTH_TOKEN: ${{ secrets.SYNAPSE_TOKEN_DPE }}
+      #SYNAPSE_UPLOAD_FOLDER_ID:
+
+    steps:
+      - uses: actions/checkout@v4
+        with: 
+          fetch-depth: 0
+      - uses: actions/setup-python@v4 
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          export PATH="$HOME/.local/bin:$PATH"
+
+      - name: Install dependencies
+        run: |
+          poetry install
+
+      - name: Generate all manifests
+        working-directory: processes
+        run: poetry run ./generate_all_templates.sh
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "GitHub Action: generate latest manifests"
+          title: "Update manifests"
+          body: |
+            This PR updates the manifests based on the latest changes.
+            Please review and merge if everything looks correct.
+          branch: update-manifests
+          base: main
+          labels: |
+            automated pr
+            manifest update
+          reviewers: $$ {github.actor }}
+          draft: false
+          delete-branch: true
+
+      #- name: Upload all manifests
+      #  working-directory: elite-data/manifest-templates
+      #  run: poetry run ../../processes/upload_templates.sh

processes/generate_all_templates.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# generate GoogleSheets templates
+
+set -e
+set -o pipefail
+
+TEST_CONFIG_PATH=../dca-template-config.json
+TEST_CONFIG=dca-template-config.json
+SCHEMATIC_CONFIG_PATH=../schematic_config.yml
+SCHEMATIC_CONFIG=schematic_config.yml
+CREDS_PATH=../schematic_service_account_creds.json
+CREDS=./schematic_service_account_creds.json
+DATA_MODEL_PATH=../EL.data.model.jsonld
+DATA_MODEL=EL.data.model.jsonld
+EXCEL_DIR=../elite-data/manifest-templates
+JSON_DIR=../elite-data/current-manifest-schemas
+LOG_DIR=../elite-data/logs
+SLEEP_THROTTLE=45 # API rate-limiting, need to better figure out dynamically based on # of templates
+
+# copy schematic-config.yml into tests/ 
+cp $SCHEMATIC_CONFIG_PATH $SCHEMATIC_CONFIG
+echo "✓ Using schematic configuration settings from $SCHEMATIC_CONFIG"
+
+# Setup for creds
+if [ -f "$CREDS_PATH" ]; then
+  cp $CREDS_PATH $CREDS
+fi
+
+# If testing locally, it might already be in folder; 
+# Else, especially if in Actions or Codespace, we need to create it from env var
+# See https://github.com/nf-osi/nf-metadata-dictionary/settings/secrets/codespaces
+if [ -f "$CREDS" ]; then
+  echo "✓ $CREDS -- running tests locally"
+elif [ -n "${SCHEMATIC_SERVICE_ACCOUNT_CREDS}" ]; then
+  echo "${SCHEMATIC_SERVICE_ACCOUNT_CREDS}" > $CREDS
+  echo "✓ Created temp $CREDS for test"
+else
+  echo "✗ Failed to access stored creds. Aborting test."
+  exit 1
+fi
+
+# Set up templates config
+cp $TEST_CONFIG_PATH $TEST_CONFIG
+echo "✓ Using copy of $TEST_CONFIG_PATH for test"
+
+TEMPLATES=($(jq '.manifest_schemas[] | .schema_name' $TEST_CONFIG | tr -d '"'))
+echo "✓ Using config with ${#TEMPLATES[@]} templates..."
+
+# Setup data model
+cp $DATA_MODEL_PATH $DATA_MODEL
+echo "✓ Set up $DATA_MODEL for test"
+
+# Setup logs
+mkdir -p $LOG_DIR
+
+# Setup schema dir
+mkdir -p $JSON_DIR
+
+for i in ${!TEMPLATES[@]}
+do
+  echo ">>>>>>> Generating ${TEMPLATES[$i]}"
+  schematic manifest --config "$SCHEMATIC_CONFIG" get -dt "${TEMPLATES[$i]}asdfasd" -p "$DATA_MODEL" -oxlsx "$EXCEL_DIR/EL_template_${TEMPLATES[$i]}.xlsx" | tee $LOG_DIR/${TEMPLATES[$i]%.*}_log 
+  sleep $SLEEP_THROTTLE
+done
+
+echo "Moving manifest json schemas to $JSON_DIR"
+mv *.schema.json $JSON_DIR
+
+echo "Cleaning up processes directory"
+rm -f $CREDS $TEST_CONFIG $DATA_MODEL $SCHEMATIC_CONFIG *.manifest.csv
+
+echo "✓ Done!"

processes/upload_templates.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# upload metadata templates to synapse
+# if running locally set SYNAPSE_AUTH_TOKEN in your environment
+
+set -e
+shopt -s extglob
+
+echo Uploading manifests to "${SYNAPSE_UPLOAD_FOLDER_ID}"
+
+MANIFESTS=("$PWD"/*.@(xlsx|xls))
+
+for MANIFEST in ${MANIFESTS[@]};
+do
+  echo "Uploading $MANIFEST"
+  synapse store --parentId "${SYNAPSE_UPLOAD_FOLDER_ID}" --noForceVersion "$MANIFEST"
+done
+
+echo "✓ Done!"