From fd06e4afa1256ec67ac9f82f3f818b2844504442 Mon Sep 17 00:00:00 2001
From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:05:14 +0000
Subject: [PATCH 1/5] Cloud tests switched to using a matrix strategy

---
 .github/workflows/cloud_tests_full.yml | 121 +++++++++++--------------
 1 file changed, 52 insertions(+), 69 deletions(-)

diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml
index 2748b542..a04aeeb3 100644
--- a/.github/workflows/cloud_tests_full.yml
+++ b/.github/workflows/cloud_tests_full.yml
@@ -5,82 +5,65 @@ on:
     types: [published]
   workflow_dispatch:
     inputs:
-      platform:
-        description: "Platform to run test"
-        required: true
-        default: "all"
-        type: choice
-        options:
-          - all
-          - aws
-          - azure
-          - gcp
+      test:
+        description: "Test profile (smaller)"
+        type: boolean
+        default: true
+      test_full:
+        description: "Full sized test"
+        type: boolean
+        default: false
+      aws:
+        description: "AWS Batch"
+        type: boolean
+        default: true
+      azure:
+        description: "Azure Batch"
+        type: boolean
+        default: true
+
 jobs:
-  run-full-tests-on-aws:
-    if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' || !github.event.inputs }}
+  run-tests:
+    if: ${{ github.repository == 'nf-core/fetchngs' }}
     runs-on: ubuntu-latest
-    steps:
-      - uses: seqeralabs/action-tower-launch@v2
-        with:
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
-          workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
-          run_name: "aws_fetchngs_full"
-          revision: ${{ github.sha }}
-          profiles: test_full
-          parameters: |
-            {
-                "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-                "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-${{ github.sha }}/"
-            }
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Tower debug log file
-          path: tower_action_*.log
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - profile: test
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
+          - profile: test
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
+          - profile: test_full
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
+          - profile: test_full
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
 
-  run-full-tests-on-azure:
-    if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' || !github.event.inputs }}
-    runs-on: ubuntu-latest
     steps:
-      - uses: seqeralabs/action-tower-launch@v2
+      - name: Launch
+        uses: seqeralabs/action-tower-launch@v2
+        # If inputs item contains the enabled profile (test or test_full)
+        # If inputs item contains the cloud provider (aws or azure)
+        # If is false, we negate and run the job
+        if: ( !contains(inputs[matrix.profile], 'false') && !contains(inputs[matrix.cloud], 'false') )
         with:
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
-          workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}"
-          run_name: "azure_fetchngs_full"
+          run_name: "fetchngs_${{ matrix.profile }}_${{ matrix.cloud}}"
+          profiles: ${{ matrix.profile }}
           revision: ${{ github.sha }}
-          profiles: test_full
-          parameters: |
-            {
-                "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-                "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-${{ github.sha }}/"
-            }
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Tower debug log file
-          path: tower_action_*.log
-
-  run-full-tests-on-gcp:
-    if: ${{ github.event.inputs.platform == 'gcp' || !github.event.inputs }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: seqeralabs/action-tower-launch@v2
-        with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_CE_GCP_CPU }}
-          workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}"
-          run_name: "gcp_fetchngs_full"
-          revision: ${{ github.sha }}
-          profiles: test_full
+          compute_env: ${{ secrets[matrix.compute_env] }}
+          workdir: ${{ secrets[matrix.workdir] }}/work/fetchngs/work-${{ github.sha }}/${{ matrix.profile }}
           parameters: |
             {
-                "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-                "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-${{ github.sha }}/"
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "outdir": "${{ secrets[matrix.workdir] }}/fetchngs/results-${{ github.sha }}/${{ matrix.profile }}/"
             }
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Tower debug log file
-          path: tower_action_*.log
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}

From 63860b198d4edc10e2e4469f2de655efd9ea5208 Mon Sep 17 00:00:00 2001
From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:12:43 +0000
Subject: [PATCH 2/5] Expand matrix to include download methods

---
 .github/workflows/cloud_tests_full.yml | 42 ++++++++++++++++++++------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml
index a04aeeb3..9d9fde9b 100644
--- a/.github/workflows/cloud_tests_full.yml
+++ b/.github/workflows/cloud_tests_full.yml
@@ -21,6 +21,18 @@ on:
         description: "Azure Batch"
         type: boolean
         default: true
+      aspera:
+        description: "Use Aspera to download"
+        type: boolean
+        default: true
+      ftp:
+        description: "Use FTP to download"
+        type: boolean
+        default: true
+      sratools:
+        description: "Use SRA tools to download"
+        type: boolean
+        default: true
 
 jobs:
   run-tests:
@@ -29,21 +41,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        profile:
+          - test
+          - test_full
+        method:
+          - aspera
+          - ftp
+          - sratools
         include:
-          - profile: test
-            cloud: aws
+          - cloud: aws
             compute_env: TOWER_COMPUTE_ENV
             workdir: TOWER_BUCKET_AWS
-          - profile: test
-            cloud: azure
+          - cloud: azure
             compute_env: TOWER_CE_AZURE_CPU
             workdir: TOWER_BUCKET_AZURE
-          - profile: test_full
-            cloud: aws
+          - cloud: aws
             compute_env: TOWER_COMPUTE_ENV
             workdir: TOWER_BUCKET_AWS
-          - profile: test_full
-            cloud: azure
+          - cloud: azure
             compute_env: TOWER_CE_AZURE_CPU
             workdir: TOWER_BUCKET_AZURE
 
@@ -55,7 +70,7 @@ jobs:
         # If is false, we negate and run the job
         if: ( !contains(inputs[matrix.profile], 'false') && !contains(inputs[matrix.cloud], 'false') )
         with:
-          run_name: "fetchngs_${{ matrix.profile }}_${{ matrix.cloud}}"
+          run_name: "fetchngs_${{ matrix.profile }}_${{ matrix.method }}-${{ matrix.cloud}}"
           profiles: ${{ matrix.profile }}
           revision: ${{ github.sha }}
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
@@ -67,3 +82,12 @@ jobs:
               "outdir": "${{ secrets[matrix.workdir] }}/fetchngs/results-${{ github.sha }}/${{ matrix.profile }}/"
             }
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+
+      - uses: actions/upload-artifact@v3
+        name: Save Logs
+        if: success() || failure()
+        with:
+          name: tower-${{ matrix.profile }}-${{ matrix.cloud }}-${{ matrix.method }}-log
+          path: |
+            tower_action_*.log
+            tower_action_*.json

From b7e626b23c78750eed79e7b0f3b498413068e98b Mon Sep 17 00:00:00 2001
From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Thu, 22 Feb 2024 13:59:10 +0000
Subject: [PATCH 3/5] Remove duplicate cloud matrix templates

---
 .github/workflows/cloud_tests_full.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml
index 9d9fde9b..61cfdd6f 100644
--- a/.github/workflows/cloud_tests_full.yml
+++ b/.github/workflows/cloud_tests_full.yml
@@ -55,12 +55,6 @@ jobs:
           - cloud: azure
             compute_env: TOWER_CE_AZURE_CPU
             workdir: TOWER_BUCKET_AZURE
-          - cloud: aws
-            compute_env: TOWER_COMPUTE_ENV
-            workdir: TOWER_BUCKET_AWS
-          - cloud: azure
-            compute_env: TOWER_CE_AZURE_CPU
-            workdir: TOWER_BUCKET_AZURE
 
     steps:
       - name: Launch

From 947a986ef7b77a1fdbfb6ce3a6cbbbc113b798a6 Mon Sep 17 00:00:00 2001
From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Thu, 22 Feb 2024 13:59:10 +0000
Subject: [PATCH 4/5] Remove duplicate cloud matrix templates

---
 .github/workflows/cloud_tests_full.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml
index 9d9fde9b..61cfdd6f 100644
--- a/.github/workflows/cloud_tests_full.yml
+++ b/.github/workflows/cloud_tests_full.yml
@@ -55,12 +55,6 @@ jobs:
           - cloud: azure
             compute_env: TOWER_CE_AZURE_CPU
             workdir: TOWER_BUCKET_AZURE
-          - cloud: aws
-            compute_env: TOWER_COMPUTE_ENV
-            workdir: TOWER_BUCKET_AWS
-          - cloud: azure
-            compute_env: TOWER_CE_AZURE_CPU
-            workdir: TOWER_BUCKET_AZURE
 
     steps:
       - name: Launch

From 3f568be15a3345d5862bdac2a8a16e0f15576c14 Mon Sep 17 00:00:00 2001
From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Thu, 22 Feb 2024 14:24:03 +0000
Subject: [PATCH 5/5] Name tests correctly

---
 .github/workflows/cloud_tests_full.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml
index 61cfdd6f..09f6de1f 100644
--- a/.github/workflows/cloud_tests_full.yml
+++ b/.github/workflows/cloud_tests_full.yml
@@ -36,6 +36,7 @@ on:
 
 jobs:
   run-tests:
+    name: ${{ matrix.profile }}-${{ matrix.method }}-${{ matrix.cloud }}
     if: ${{ github.repository == 'nf-core/fetchngs' }}
     runs-on: ubuntu-latest
     strategy:
@@ -62,9 +63,9 @@ jobs:
         # If inputs item contains the enabled profile (test or test_full)
         # If inputs item contains the cloud provider (aws or azure)
         # If is false, we negate and run the job
-        if: ( !contains(inputs[matrix.profile], 'false') && !contains(inputs[matrix.cloud], 'false') )
+        if: ( !contains(inputs[matrix.profile], 'false') && !contains(inputs[matrix.cloud], 'false') && !contains(inputs[matrix.method], 'false') )
         with:
-          run_name: "fetchngs_${{ matrix.profile }}_${{ matrix.method }}-${{ matrix.cloud}}"
+          run_name: "fetchngs_${{ matrix.profile }}_${{ matrix.method }}_${{ matrix.cloud}}"
           profiles: ${{ matrix.profile }}
           revision: ${{ github.sha }}
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
@@ -73,7 +74,8 @@ jobs:
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "${{ secrets[matrix.workdir] }}/fetchngs/results-${{ github.sha }}/${{ matrix.profile }}/"
+              "outdir": "${{ secrets[matrix.workdir] }}/fetchngs/results-${{ github.sha }}/${{ matrix.profile }}/",
+              ""
             }
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}