Create self-hosted runner for integration(-ish) CI tests

.github/workflows/ci.yml

@@ -197,9 +197,16 @@ jobs:
               (cd $dir && terraform plan -input=false)
           done
 
+  provion-self-hosted-runner:
+    name: provion-self-hosted-runner
+    uses: ./.github/workflows/deploy-self-hosted-runner.yml
+    secrets: inherit
+
   k3d_test:
     name: k3d_test
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    needs:
+      - provion-self-hosted-runner
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -221,3 +228,9 @@ jobs:
               (cd $dir && terraform validate)
               (cd $dir && terraform plan -input=false)
           done
+
+  destory-self-hosted-runner:
+    name: destory-self-hosted-runner
+    needs: k3d_test
+    if: always()
+    uses: ./.github/workflows/destroy-self-hosted-runner.yml

.github/workflows/deploy-self-hosted-runner.yml

@@ -0,0 +1,64 @@
+name: Deploy the test runner vm to azure
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - develop
+    paths-ignore: ["**.md"]
+
+jobs:
+  deploy_test_vm:
+    name: Deploy VM to azure
+    runs-on: ubuntu-latest
+    env:
+        ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+        ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+        ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+        ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+    permissions:
+      contents: "read"
+      id-token: "write"
+
+    defaults:
+      run:
+        working-directory: ./infrastructure
+
+    steps:
+      - name: Checkout the Code
+        uses: actions/checkout@v3
+
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+      - name: Login to Azure
+        run: |
+          az login --service-principal --username $ARM_CLIENT_ID --password $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - run: terraform apply -auto-approve
+        env:
+          TF_VAR_github_runner_token: ${{ secrets.runner_token }}
+
+      - name: Create blob
+        run: |
+          echo "Creating blob..."
+          az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block --data "${{ github.run_id }}"

.github/workflows/destroy-self-hosted-runner.yml

@@ -0,0 +1,58 @@
+name: Destroy the test runner vm to azure
+
+on:
+    workflow_call:
+    workflow_dispatch:
+
+jobs:
+  destroy_test_vm:
+    name: Destroy VM to azure
+    runs-on: ubuntu-latest
+    env:
+        ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+        ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+        ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+        ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+    permissions:
+      contents: "read"
+      id-token: "write"
+
+    defaults:
+      run:
+        working-directory: ./infrastructure
+
+    steps:
+      - name: Checkout the Code
+        uses: actions/checkout@v3
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - name: Delete blob
+        run: |
+          az storage blob delete --account-name zenmlstorageaccount --container-name github-runner-tf --name ${{ github.run_id }}
+
+      - name: Check if any blobs left
+        id: check_blobs
+        run: |
+          blobs=$(az storage blob list --account-name zenmlstorageaccount --container-name github-runner-tf --query "[?starts_with(name, 'github-run')].name" --output tsv)
+          echo "BLOBS=$blobs" >> $GITHUB_ENV
+
+      - name: Destroy VM
+        run: terraform destroy -auto-approve -refresh=False
+        env:
+          TF_VAR_github_runner_token: ${{ secrets.runner_token }}
+        if: env.BLOBS == ''

infrastructure/deploy.tf

@@ -0,0 +1,104 @@
+resource "azurerm_resource_group" "example" {
+  name     = "zenml-github-test"
+  location = "West Europe"
+
+  tags = {
+    z-env = "dev"
+    z-owner = "safoine-ext"
+    z-project = "testing"
+    z-team = "oss"
+    z-description = "resources for integration testing"
+  }
+}
+
+resource "azurerm_virtual_network" "example" {
+  name                = "mlstack-test-network"
+  address_space       = ["10.0.0.0/16"]
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+}
+
+resource "azurerm_subnet" "example" {
+  name                 = "mlstack-subnet"
+  resource_group_name  = azurerm_resource_group.example.name
+  virtual_network_name = azurerm_virtual_network.example.name
+  address_prefixes     = ["10.0.2.0/24"]
+}
+
+resource "azurerm_network_interface" "example" {
+  name                = "mlstack-nic"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+
+  ip_configuration {
+    name                          = "mlstack-ip"
+    subnet_id                     = azurerm_subnet.example.id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id          = azurerm_public_ip.example.id
+  }
+}
+
+resource "azurerm_public_ip" "example" {
+  name                = "mlstack-pip"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+  allocation_method   = "Dynamic"
+}
+
+resource "azurerm_network_security_group" "example" {
+  name                = "mlstack-nsg"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+}
+
+resource "azurerm_network_security_rule" "example" {
+  name                        = "SSH"
+  priority                    = 1001
+  direction                   = "Inbound"
+  access                      = "Allow"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "22"
+  source_address_prefix       = "*"
+  destination_address_prefix  = "*"
+  resource_group_name         = azurerm_resource_group.example.name
+  network_security_group_name = azurerm_network_security_group.example.name
+}
+
+resource "azurerm_network_interface_security_group_association" "example" {
+  network_interface_id      = azurerm_network_interface.example.id
+  network_security_group_id = azurerm_network_security_group.example.id
+}
+
+data "azurerm_ssh_public_key" "example" {
+  name                = "mlstack-test-vm"
+  resource_group_name = "zenml-developers"
+}
+
+data "azurerm_image" "example" {
+  name                = "mlstack-github-runner-machine-image-20230819162059"
+  resource_group_name = "zenml-developers"
+}
+
+resource "azurerm_linux_virtual_machine" "example" {
+  name                = "mlstack-test-machine"
+  resource_group_name = azurerm_resource_group.example.name
+  location            = azurerm_resource_group.example.location
+  size                = "Standard_D8s_v3"
+  admin_username      = "mlstackuser"
+  network_interface_ids = [
+    azurerm_network_interface.example.id,
+  ]
+
+  admin_ssh_key {
+    username   = "mlstackuser"
+    public_key = data.azurerm_ssh_public_key.example.public_key
+  }
+
+  os_disk {
+    caching              = "ReadWrite"
+    storage_account_type = "StandardSSD_LRS"
+  }
+
+  source_image_id = data.azurerm_image.example.id
+}

infrastructure/terraform.tf

@@ -0,0 +1,22 @@
+# defining the providers for the recipe module
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">=3.16.0"
+    }
+  }
+
+  required_version = ">= 0.14.8"
+
+  backend "azurerm" {
+    resource_group_name  = "zenml-developers"
+    storage_account_name = "zenmlstorageaccount"
+    container_name       = "github-runner-tf"
+    key                  = "terraform.tfstate"
+  }
+}
+
+provider "azurerm" {
+  features {}
+}

infrastructure/variables.tf

@@ -0,0 +1,4 @@
+variable "github_runner_token" {
+  description = "GitHub token"
+  type        = string
+}