Create self-hosted runner for integration(-ish) CI tests

.github/workflows/ci.yml

@@ -193,9 +193,16 @@ jobs:
               (cd $dir && terraform plan -input=false)
           done
 
+  provion-self-hosted-runner:
+    name: provion-self-hosted-runner
+    uses: ./.github/workflows/deploy-self-hosted-runner.yml
+    secrets: inherit
+
   k3d_test:
     name: k3d_test
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    needs:
+      - provion-self-hosted-runner
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -217,3 +224,9 @@ jobs:
               (cd $dir && terraform validate)
               (cd $dir && terraform plan -input=false)
           done
+
+  destory-self-hosted-runner:
+    name: destory-self-hosted-runner
+    needs: k3d_test
+    if: always()
+    uses: ./.github/workflows/destroy-self-hosted-runner.yml

.github/workflows/deploy-test-runner.yml → .github/workflows/deploy-self-hosted-runner.yml

@@ -18,7 +18,6 @@ jobs:
         ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
         ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
         ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
-        MLSTACKS_ANALYTICS_OPT_IN=false
     permissions:
       contents: "read"
       id-token: "write"
@@ -31,6 +30,14 @@ jobs:
       - name: Checkout the Code
         uses: actions/checkout@v3
 
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+      - name: Login to Azure
+        run: |
+          az login --service-principal --username $ARM_CLIENT_ID --password $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
+
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v2
 
@@ -49,4 +56,9 @@ jobs:
 
       - run: terraform apply -auto-approve
         env:
-          TF_VAR_github_runner_token: ${{ secrets.runner_token }}
+          TF_VAR_github_runner_token: ${{ secrets.runner_token }}
+
+      - name: Create blob
+        run: |
+          echo "Creating blob..."
+          az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block --data "${{ github.run_id }}"

.github/workflows/destroy-test-runner.yml → .github/workflows/destroy-self-hosted-runner.yml

@@ -13,7 +13,6 @@ jobs:
         ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
         ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
         ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
-        MLSTACKS_ANALYTICS_OPT_IN=false
     permissions:
       contents: "read"
       id-token: "write"
@@ -42,6 +41,18 @@ jobs:
         id: validate
         run: terraform validate -no-color
 
-      - run: terraform destroy -auto-approve -refresh=False
+      - name: Delete blob
+        run: |
+          az storage blob delete --account-name zenmlstorageaccount --container-name github-runner-tf --name ${{ github.run_id }}
+
+      - name: Check if any blobs left
+        id: check_blobs
+        run: |
+          blobs=$(az storage blob list --account-name zenmlstorageaccount --container-name github-runner-tf --query "[?starts_with(name, 'github-run')].name" --output tsv)
+          echo "BLOBS=$blobs" >> $GITHUB_ENV
+
+      - name: Destroy VM
+        run: terraform destroy -auto-approve -refresh=False
         env:
-          TF_VAR_github_runner_token: ${{ secrets.runner_token }}
+          TF_VAR_github_runner_token: ${{ secrets.runner_token }}
+        if: env.BLOBS == ''

infrastructure/deploy.tf

@@ -1,6 +1,14 @@
 resource "azurerm_resource_group" "example" {
   name     = "zenml-github-test"
   location = "West Europe"
+
+  tags = {
+    z-env = "dev"
+    z-owner = "safoine-ext"
+    z-project = "testing"
+    z-team = "oss"
+    z-description = "resources for integration testing"
+  }
 }
 
 resource "azurerm_virtual_network" "example" {

infrastructure/terraform.tf

@@ -19,13 +19,4 @@ terraform {
 
 provider "azurerm" {
   features {}
-  default_tags {
-    tags = {
-      z-env = "dev"
-      z-owner       = "safoine-ext"
-      z-project = "testing"
-      z-team = "oss"
-      z-description = "resources for integration testing"
-    }
-  }
 }