From 33bd8d5a330c181841e7c9f0ea5f1c2ed2876f73 Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <MukulChandrakant.Mahadik@nrel.gov>
Date: Fri, 20 Sep 2024 20:42:30 -0700
Subject: [PATCH 1/7] Task A-6: Switching to step outputs instead of GITHUB_ENV

Refer to details in cleanup issue:
Task A-6: https://github.com/e-mission/e-mission-docs/issues/1082#issuecomment-2364335425
---
 .github/workflows/image_build_push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml
index 1847af67b..8580a0b03 100644
--- a/.github/workflows/image_build_push.yml
+++ b/.github/workflows/image_build_push.yml
@@ -24,7 +24,7 @@ jobs:
 
     - name: Get current date # get the date of the build
       id: date
-      run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')"
+      run: echo "date=$(date +'%Y-%m-%d--%M-%S')" >> "$GITHUB_OUTPUT"
 
     #Runs a single command using the runners shell
     - name: Run a one-line script

From 96a8213c14019ddfd4e7c5635c4cf9202964ff0a Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <MukulChandrakant.Mahadik@nrel.gov>
Date: Fri, 20 Sep 2024 21:28:12 -0700
Subject: [PATCH 2/7] Task A-2: Storing latest tag in .env file + Task A-8:
 Prefix branch name

Refer to details in cleanup issue:
Task A-2: https://github.com/e-mission/e-mission-docs/issues/1082#issuecomment-2364583414

Storing server tag as well so that artifacts are not needed.
Can also remove image tag passed as input in Workflow dispatch POST request.
Workflow input also removed in dashboard workflows

For now not removing artifacts until the internal script is updated to handle this change.

----

Task A-8: Prefixing branch name to the docker tag along with the date.
In the internal script we will not need to maintain the different branch lists as the images will be completely tagged in the external workflows themselves.
We can simply use the tags without modifications then.

For now, not prefixing the tag to the artifact since we will be removing the artifact anyways.
And current internal script works with artifacts.
Once I update the internal script, will come back and remove artifacts.
---
 .github/workflows/image_build_push.yml | 30 +++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml
index 8580a0b03..4f22ab7f0 100644
--- a/.github/workflows/image_build_push.yml
+++ b/.github/workflows/image_build_push.yml
@@ -18,6 +18,14 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
+    - name: Set docker image tags
+      id: set-tags
+      run: |
+        set -a; source .env; set +a
+        echo "SERVER_IMAGE_TAG=${SERVER_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+        echo "Current server image tag (push): ${SERVER_IMAGE_TAG}"
+  
     - name: docker login
       run: | # log into docker hub account
         docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
@@ -40,6 +48,22 @@ jobs:
       run: |
         docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }}
 
+    - name: Update .env file 
+      run: |
+        echo "SERVER_IMAGE_TAG=${GITHUB_REF##*/}_${{ steps.date.outputs.date }}" > .env
+
+    - name: Add, Commit, Push changes to .env file
+      run: |
+        git config --local user.email "action@github.com"
+        git config --local user.name "Github Actions bot to update .env with latest tags"
+        if git diff --quiet; then
+          echo "Latest timestamp already present in .env file, no changes to commit"
+        else
+          git add .env
+          git commit -m "Updated docker image tags in .env file to the latest timestamp"
+          git push origin 
+        fi
+
     - name: Create a text file
       run: |
         echo ${{ steps.date.outputs.date }} > tag_file.txt
@@ -56,9 +80,6 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
 
-    env:
-      DOCKER_IMAGE_TAG: ${{ needs.build.outputs.date }}
-
     strategy:
       matrix:
         include:
@@ -71,7 +92,6 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Trigger workflow in admin-dash, public-dash
-      # TODO: Create Fine-grained token with "Actions: write" permissions 
       run: |
         curl -L \
           -X POST \
@@ -79,4 +99,4 @@ jobs:
           -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \
-          -d '{"ref":"${{ matrix.branch }}", "inputs": {"docker_image_tag" : "${{ env.DOCKER_IMAGE_TAG }}"}}'
+          -d '{"ref":"${{ matrix.branch }}"}'

From c1917c4118811bed9439996cdf223b6cba3b300d Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <MukulChandrakant.Mahadik@nrel.gov>
Date: Fri, 20 Sep 2024 21:44:11 -0700
Subject: [PATCH 3/7] Task A-7: Removed certificates from external Dockerfile;
 added internally

Refer to issue comment for details:
Task A-7: https://github.com/e-mission/e-mission-docs/issues/1082#issuecomment-2364315699

The certificates are relevant to our internal AWS configuration and not needed externally.
They can be present externally too without having any major effect.
But removing them helps keeping the base image clean.
Additionally, anyone working with the code can customize with their own certificates if needed or adopt an approach which doesn't even need certificates in the first place.
---
 Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index fcea642fd..04dba77f0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,8 +3,6 @@ FROM ubuntu:jammy-20240227
 
 MAINTAINER K. Shankari (shankari@eecs.berkeley.edu)
 
-ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /etc/ssl/certs/
-
 WORKDIR /usr/src/app
 
 RUN apt-get -y -qq update

From f2f344c452c9158f748c7dc6ae058901cc091600 Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <MukulChandrakant.Mahadik@nrel.gov>
Date: Fri, 20 Sep 2024 21:59:24 -0700
Subject: [PATCH 4/7] Task A-2: Added .env file initialized with the current
 latest tag

---
 .env | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .env

diff --git a/.env b/.env
new file mode 100644
index 000000000..b290898cf
--- /dev/null
+++ b/.env
@@ -0,0 +1 @@
+SERVER_IMAGE_TAG=2024-09-20--06-45

From 3a01028463accf790567d7ab55b70a51becd2f77 Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <MukulChandrakant.Mahadik@nrel.gov>
Date: Mon, 23 Sep 2024 14:39:54 -0700
Subject: [PATCH 5/7] Task A-2: Removed artifact upload ; internal script
 updated

Internal script updated as well.
Internal PR must be merged as well once these external PR changes merged.
---
 .github/workflows/image_build_push.yml | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml
index 4f22ab7f0..1afab6fca 100644
--- a/.github/workflows/image_build_push.yml
+++ b/.github/workflows/image_build_push.yml
@@ -64,18 +64,6 @@ jobs:
           git push origin 
         fi
 
-    - name: Create a text file
-      run: |
-        echo ${{ steps.date.outputs.date }} > tag_file.txt
-        echo "Created tag text file"
-
-    - name: Upload Artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: docker-image-tag
-        path: tag_file.txt
-        overwrite: true
-
   dispatch:
     needs: build
     runs-on: ubuntu-latest

From b2382ec6ed26099486379aafa57f5bea9e828074 Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <mmahadik@asu.edu>
Date: Sun, 29 Sep 2024 12:01:52 -0700
Subject: [PATCH 6/7] Task A-5: Added reusable workflow
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Storing a reusable workflow in the e-mission-server repo. Can decide where to place it in a central location.
https://docs.github.com/en/actions/sharing-automations/reusing-workflows

It essentially works like a function call in normal programming.
The advantage is that we have no repeated code the image build process.
All the other repos (join, admin-dash, public-dash) reuse the same workflow file.

Additionally, on for future GitHub actions, workflow file related changes, will no longer need to have 3 additional PRs for each repo (join, admin-dash, public-dash). Can simply modify the reusable workflow file as this is the core “function” workflow that is being called.

I have added conditional checks that check for the repo name in the reusable workflow file that determine which statements to execute depending on for which repo the workflow is running.

This is used for both push events specific to a repo as well as for the workflow dispatch events triggered on pushes to server repo.
---
 .../workflows/reusable_image_build_push.yml   | 141 ++++++++++++++++++
 1 file changed, 141 insertions(+)
 create mode 100644 .github/workflows/reusable_image_build_push.yml

diff --git a/.github/workflows/reusable_image_build_push.yml b/.github/workflows/reusable_image_build_push.yml
new file mode 100644
index 000000000..6bbfcf8e2
--- /dev/null
+++ b/.github/workflows/reusable_image_build_push.yml
@@ -0,0 +1,141 @@
+name: Reusable Docker Image Build and Push
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        required: true
+        type: string
+      branch:
+        required: true
+        type: string
+
+env:
+  DOCKER_USER: ${{secrets.DOCKER_USER}}
+  DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: e-mission/${{ inputs.repo }}
+          ref: ${{ inputs.branch }}
+          token: ${{ secrets.GH_FG_PAT_TAGS }}
+    
+      - name: Fetch server image tag
+        id: get-server-tag
+        run: |
+          if [ "${{ inputs.repo }}" = "op-admin-dashboard" ] || [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then
+            response=$(curl -s https://raw.githubusercontent.com/e-mission/e-mission-server/refs/heads/cleanup-cicd/.env)
+            SERVER_IMAGE_TAG=$(echo "$response" | grep "SERVER_IMAGE_TAG=" | cut -d'=' -f2)
+            echo "SERVER_IMAGE_TAG=$SERVER_IMAGE_TAG" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set docker image tags
+        id: set-tags
+        run: |
+          ls -al
+          set -a; source .env; set +a
+          if [ "${{ inputs.repo }}" = "nrel-openpath-join-page" ]; then
+            echo "JOIN_IMAGE_TAG=${JOIN_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+          elif [ "${{ inputs.repo }}" = "op-admin-dashboard" ]; then
+            echo "ADMIN_DASH_IMAGE_TAG=${ADMIN_DASH_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+          elif [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then
+            echo "PUBLIC_DASH_NOTEBOOK_IMAGE_TAG=${PUBLIC_DASH_NOTEBOOK_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+            echo "PUBLIC_DASH_FRONTEND_IMAGE_TAG=${PUBLIC_DASH_FRONTEND_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+          fi
+         
+      - name: Print input docker image tags
+        run: |
+          echo "Event name: ${{ github.event_name }}"
+          if [ "${{ inputs.repo }}" = "nrel-openpath-join-page" ]; then
+            echo "Current join-page image tag: ${{ steps.set-tags.outputs.JOIN_IMAGE_TAG }}"
+          elif [ "${{ inputs.repo }}" = "op-admin-dashboard" ]; then
+            echo "Current admin-dash image tag: ${{ steps.set-tags.outputs.ADMIN_DASH_IMAGE_TAG }}"
+            echo "Latest server image tag (${{ github.event_name }}): ${{ steps.get-server-tag.outputs.SERVER_IMAGE_TAG }}"
+          elif [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then            
+            echo "Current notebook image tag (push): ${{ steps.set-tags.outputs.PUBLIC_DASH_NOTEBOOK_IMAGE_TAG }}"
+            echo "Current frontend image tag (push): ${{ steps.set-tags.outputs.PUBLIC_DASH_FRONTEND_IMAGE_TAG }}"
+            echo "Latest server image tag (${{ github.event_name }}): ${{ steps.get-server-tag.outputs.SERVER_IMAGE_TAG }}"
+          fi
+
+      - name: docker login
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+      - name: Get current date # get the date of the build
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d--%M-%S')" >> "$GITHUB_OUTPUT"
+
+      - name: Run a one-line script
+        run: echo running image build for repo ${{ inputs.repo }} branch ${{ inputs.branch }} on ${{ steps.date.outputs.date }}
+
+      - name: build docker image
+        run: |
+          if [ "${{ inputs.repo }}" = "nrel-openpath-join-page" ]; then
+            docker build -t $DOCKER_USER/${{ inputs.repo }}:${{ inputs.branch }}_${{ steps.date.outputs.date }} ./frontend
+          elif [ "${{ inputs.repo }}" = "op-admin-dashboard" ]; then
+            SERVER_IMAGE_TAG=${{ steps.get-server-tag.outputs.SERVER_IMAGE_TAG }} docker compose -f docker-compose-prod.yml build
+          elif [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then            
+            SERVER_IMAGE_TAG=${{ steps.get-server-tag.outputs.SERVER_IMAGE_TAG }} docker compose -f docker-compose.yml build
+          fi
+          docker images
+
+      - name: rename docker image
+        run: |
+          if [ "${{ inputs.repo }}" = "op-admin-dashboard" ]; then
+            docker image tag e-mission/opdash:0.0.1 $DOCKER_USER/${{ inputs.repo }}:${{ inputs.branch }}_${{ steps.date.outputs.date }}
+          elif [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then      
+            if [ "${{ github.event_name }}" == "push" ]; then
+              docker image tag em-pub-dash-prod/frontend:latest $DOCKER_USER/${{ inputs.repo }}_frontend:${{ inputs.branch }}_${{ steps.date.outputs.date }}
+            fi
+            docker image tag em-pub-dash-prod/viz-scripts:latest $DOCKER_USER/${{ inputs.repo }}_notebook:${{ inputs.branch }}_${{ steps.date.outputs.date }}
+          fi
+  
+      - name: push docker image
+        run: |
+          if [ "${{ inputs.repo }}" = "op-admin-dashboard" ] || [ "${{ inputs.repo }}" = "nrel-openpath-join-page" ]; then
+            docker push $DOCKER_USER/${{ inputs.repo }}:${{ inputs.branch }}_${{ steps.date.outputs.date }}
+          elif [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then      
+            if [ "${{ github.event_name }}" == "push" ]; then
+              docker push $DOCKER_USER/${{ inputs.repo }}_frontend:${{ inputs.branch }}_${{ steps.date.outputs.date }}
+            fi
+            docker push $DOCKER_USER/${{ inputs.repo }}_notebook:${{ inputs.branch }}_${{ steps.date.outputs.date }}
+          fi
+        
+      - name: Update .env file 
+        run: |
+          if [ "${{ inputs.repo }}" = "nrel-openpath-join-page" ]; then
+            echo "JOIN_IMAGE_TAG=${{ inputs.branch }}_${{ steps.date.outputs.date }}" > .env
+          elif [ "${{ inputs.repo }}" = "op-admin-dashboard" ]; then
+            echo "ADMIN_DASH_IMAGE_TAG=${{ inputs.branch }}_${{ steps.date.outputs.date }}" > .env
+            echo "SERVER_IMAGE_TAG=${{ steps.get-server-tag.outputs.SERVER_IMAGE_TAG }}" >> .env
+          elif [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then
+            echo "PUBLIC_DASH_NOTEBOOK_IMAGE_TAG=${{ inputs.branch }}_${{ steps.date.outputs.date }}" > .env
+            if [ "${{ github.event_name }}" == "push" ]; then
+              echo "Push event: Update frontend image tag"
+              echo "PUBLIC_DASH_FRONTEND_IMAGE_TAG=${{ inputs.branch }}_${{ steps.date.outputs.date }}" >> .env
+            else
+              echo "Workflow_dispatch: Reuse existing frontend image tag"
+              echo "PUBLIC_DASH_FRONTEND_IMAGE_TAG=${{ steps.set-tags.outputs.PUBLIC_DASH_FRONTEND_IMAGE_TAG }}" >> .env
+            fi
+            echo "SERVER_IMAGE_TAG=${{ steps.get-server-tag.outputs.SERVER_IMAGE_TAG }}" >> .env
+          fi
+          cat .env
+
+      - name: Add, Commit, Push changes to .env file
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "Github Actions bot to update .env with latest tags"
+          # echo ${{ github.actor }}
+          if git diff --quiet; then
+            echo "Latest timestamp already present in .env file, no changes to commit"
+          else
+            git add .env
+            git commit -m "Updated docker image tags in .env file to the latest timestamp"
+            git push origin
+          fi
+          cat .env

From d56f6528882d3c050ed5b14103bc326715e535db Mon Sep 17 00:00:00 2001
From: "Mahadik, Mukul Chandrakant" <mmahadik@asu.edu>
Date: Sun, 29 Sep 2024 16:40:25 -0700
Subject: [PATCH 7/7] Task A-4: Run image_build_push only after tests pass
 successfully

Approach:
Converted test-with-docker and test-with-manual-install to reusable workflows.
Added them as jobs in image_build_push workflow.
Build job in this workflow needs these test jobs as dependencies.

-------

Test Workflow run
https://github.com/MukuFlash03/e-mission-server/actions/runs/11096277197/job/30825929886

-----

Notes

Github actions didn't have out of the box solution for running a workflow based on results of multiple workflows where ALL workflows must have completed successfully.
We need this since both the test-with-docker and test-with-manual-install must pass.
So this needs an "AND" logic.

"workflow_run" is there but this triggers the dependent workflow when either of the workflow dependencies defined as prerequisites are completed.
So this has an "OR" logic.
https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run

----

Found an alternative suggestion here:
https://stackoverflow.com/a/75597437

This suggests converting the pre-requisite workflows into reusable workflows.
These workflows can then be called in jobs in the workflow that needs these workflows to be run before.
Finally, these jobs can be added as dependencies for the requisite job.

In our scenario, two new jobs are added to the image_build_push.yml for each of the two tests environments.
These will run parallelly.

Then in the build image job, these jobs are added in the "needs" field, indicating that these jobs must pass successfully before running the build job.

-------

Also corrected the branch in reusable workflow for fetching latest server image tag.
Need to update in the currently open PRs as well.

------

Also, removed the `push` trigger from the two `test` workflows since the `image_build_push` workflow would also be triggered on a `push` event which in its workflow triggers these two test workflows.
Thus if not removed, each test workflow would run twice.
---
 .github/workflows/image_build_push.yml          | 7 +++++++
 .github/workflows/reusable_image_build_push.yml | 2 +-
 .github/workflows/test-with-docker.yml          | 4 ++--
 .github/workflows/test-with-manual-install.yml  | 6 ++----
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml
index 1afab6fca..bbc092cd8 100644
--- a/.github/workflows/image_build_push.yml
+++ b/.github/workflows/image_build_push.yml
@@ -10,8 +10,15 @@ env:
   DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
 
 jobs:
+  test-with-docker:
+    uses: e-mission/e-mission-server/.github/workflows/test-with-docker.yml@master
+
+  test-with-manual-install:
+    uses: e-mission/e-mission-server/.github/workflows/test-with-manual-install.yml@master
+
   build:
     runs-on: ubuntu-latest
+    needs: [test-with-docker, test-with-manual-install]
 
     outputs:
       date: ${{ steps.date.outputs.date }}
diff --git a/.github/workflows/reusable_image_build_push.yml b/.github/workflows/reusable_image_build_push.yml
index 6bbfcf8e2..5de4eb7eb 100644
--- a/.github/workflows/reusable_image_build_push.yml
+++ b/.github/workflows/reusable_image_build_push.yml
@@ -29,7 +29,7 @@ jobs:
         id: get-server-tag
         run: |
           if [ "${{ inputs.repo }}" = "op-admin-dashboard" ] || [ "${{ inputs.repo }}" = "em-public-dashboard" ]; then
-            response=$(curl -s https://raw.githubusercontent.com/e-mission/e-mission-server/refs/heads/cleanup-cicd/.env)
+            response=$(curl -s https://raw.githubusercontent.com/e-mission/e-mission-server/refs/heads/master/.env)
             SERVER_IMAGE_TAG=$(echo "$response" | grep "SERVER_IMAGE_TAG=" | cut -d'=' -f2)
             echo "SERVER_IMAGE_TAG=$SERVER_IMAGE_TAG" >> "$GITHUB_OUTPUT"
           fi
diff --git a/.github/workflows/test-with-docker.yml b/.github/workflows/test-with-docker.yml
index 1b54ed5ba..d661e6a02 100644
--- a/.github/workflows/test-with-docker.yml
+++ b/.github/workflows/test-with-docker.yml
@@ -5,14 +5,14 @@ name: test-with-docker
 # Controls when the action will run. Triggers the workflow on push or pull request 
 # events but only for the master branch
 on:
-  push:
-    branches: [ master ]
   pull_request:
     branches: [ master ]
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron:  '5 4 * * 0'
 
+  workflow_call:
+
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   # This workflow contains a single job called "build"
diff --git a/.github/workflows/test-with-manual-install.yml b/.github/workflows/test-with-manual-install.yml
index 4a81eb000..75f920682 100644
--- a/.github/workflows/test-with-manual-install.yml
+++ b/.github/workflows/test-with-manual-install.yml
@@ -5,10 +5,6 @@ name: ubuntu-only-test-with-manual-install
 # Controls when the action will run. Triggers the workflow on push or pull request
 # events but only for the master branch
 on:
-  push:
-    branches: 
-        - master
-        - gis-based-mode-detection
   pull_request:
     branches: 
         - master
@@ -17,6 +13,8 @@ on:
     # * is a special character in YAML so you have to quote this string
     - cron:  '5 4 * * 0'
 
+  workflow_call:
+
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   # This workflow contains a single job called "build"