From 2630e0573cc76dccf24ad3bec50a8b3c05e277e5 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 08:27:52 +0100 Subject: [PATCH 01/20] change recipe-test to test the runner --- .github/workflows/recipe-test.yml | 90 ++----------------------------- 1 file changed, 4 insertions(+), 86 deletions(-) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index e1c1a6e9..7a41caf9 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -28,98 +28,16 @@ jobs: files: "." config: ./.typos.toml - aws_test: - name: aws_test - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} - AWS_DEFAULT_REGION: eu-central-1 - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.3.4 - - - name: Loop through subdirectories - run: | - for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "aws*"); do - (cd $dir && terraform init) - (cd $dir && terraform fmt -check) - (cd $dir && terraform validate) - (cd $dir && terraform plan -input=false) - done - - gcp_test: - name: gcp_test - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@master - - - id: 'auth' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: '${{ secrets.GCP_CREDENTIALS }}' - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v0' - - - name: Install Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.3.4 - - - name: Loop through subdirectories - run: | - for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "gcp*"); do - (cd $dir && terraform init) - (cd $dir && terraform fmt -check) - (cd $dir && terraform validate) - (cd $dir && terraform plan -input=false) - done - - azure_test: - name: azure-test - runs-on: ubuntu-latest - env: - ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} - ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} - ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} - ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.3.4 - - - name: Loop through subdirectories - run: | - for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "azure*"); do - (cd $dir && terraform init) - (cd $dir && terraform fmt -check) - (cd $dir && terraform validate) - (cd $dir && terraform plan -input=false) - done - k3d_test: name: k3d_test - runs-on: ubuntu-latest + runs-on: mlstack-test-runner steps: - name: Checkout uses: actions/checkout@v2 - - name: Install k3d - run: | - curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + #- name: Install k3d + # run: | + # curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash - name: Install Terraform uses: hashicorp/setup-terraform@v2 From fc4b3787ae0bf3d20ff4c817eaeaa68017e76d0c Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 08:32:10 +0100 Subject: [PATCH 02/20] change recipe-test to test the runner --- .github/workflows/recipe-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index 7a41caf9..6e8b33cc 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -30,7 +30,7 @@ jobs: k3d_test: name: k3d_test - runs-on: mlstack-test-runner + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v2 From 5a576e726401ced2c828a4d2e661e5072944eafa Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 08:37:17 +0100 Subject: [PATCH 03/20] add clone repo step --- .github/workflows/recipe-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index 6e8b33cc..9fb81e87 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -35,6 +35,12 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Clone repository + run: | + git clone https://github.com/safoinme/mlops-stacks.git + cd mlops-stacks + git checkout develop + #- name: Install k3d # run: | # curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash From 60e48df9e58f39dd9af0df1791e93965c3e0ef0e Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 08:42:28 +0100 Subject: [PATCH 04/20] remove cloning step --- .github/workflows/recipe-test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index 9fb81e87..6e8b33cc 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -35,12 +35,6 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Clone repository - run: | - git clone https://github.com/safoinme/mlops-stacks.git - cd mlops-stacks - git checkout develop - #- name: Install k3d # run: | # curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash From 1ea2aa807deb26ac5442a8e12ba069d6ccfa4d7e Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 12:24:32 +0100 Subject: [PATCH 05/20] initial code for creating a self-hosted runner on an azure vm to test heavy workloads --- .github/workflows/deploy-test-runner.yml | 46 +++++++++++ .github/workflows/destroy-test-runner.yml | 46 +++++++++++ .github/workflows/recipe-test.yml | 90 +++++++++++++++++++-- infrastructure/deploy.tf | 98 +++++++++++++++++++++++ infrastructure/terraform.tf | 15 ++++ infrastructure/variables.tf | 4 + 6 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/deploy-test-runner.yml create mode 100644 .github/workflows/destroy-test-runner.yml create mode 100644 infrastructure/deploy.tf create mode 100644 infrastructure/terraform.tf create mode 100644 infrastructure/variables.tf diff --git a/.github/workflows/deploy-test-runner.yml b/.github/workflows/deploy-test-runner.yml new file mode 100644 index 00000000..c9533ea3 --- /dev/null +++ b/.github/workflows/deploy-test-runner.yml @@ -0,0 +1,46 @@ +name: Deploy the test runner vm to azure + +on: + workflow_call: + workflow_dispatch: + +jobs: + deploy_test_vm: + name: Deploy VM to azure + runs-on: ubuntu-latest + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + permissions: + contents: "read" + id-token: "write" + + defaults: + run: + working-directory: ./infrastructure + + steps: + - name: Checkout the Code + uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + + - name: Terraform fmt + id: fmt + run: terraform fmt -check + continue-on-error: true + + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + + - run: terraform apply -auto-approve + env: + TF_VAR_github_runner_token: ${{ secrets.runner_token }} \ No newline at end of file diff --git a/.github/workflows/destroy-test-runner.yml b/.github/workflows/destroy-test-runner.yml new file mode 100644 index 00000000..ee5fa2e2 --- /dev/null +++ b/.github/workflows/destroy-test-runner.yml @@ -0,0 +1,46 @@ +name: Destroy the test runner vm to azure + +on: + workflow_call: + workflow_dispatch: + +jobs: + destroy_test_vm: + name: Destroy VM to azure + runs-on: ubuntu-latest + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + permissions: + contents: "read" + id-token: "write" + + defaults: + run: + working-directory: ./infrastructure + + steps: + - name: Checkout the Code + uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + + - name: Terraform fmt + id: fmt + run: terraform fmt -check + continue-on-error: true + + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + + - run: terraform destroy -auto-approve -refresh=False + env: + TF_VAR_github_runner_token: ${{ secrets.runner_token }} \ No newline at end of file diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index 6e8b33cc..cf04a9ee 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -28,22 +28,100 @@ jobs: files: "." config: ./.typos.toml - k3d_test: - name: k3d_test - runs-on: self-hosted + aws_test: + name: aws_test + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_DEFAULT_REGION: eu-central-1 + steps: - name: Checkout uses: actions/checkout@v2 - #- name: Install k3d - # run: | - # curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + - name: Install Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.3.4 + + - name: Loop through subdirectories + run: | + for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "aws*"); do + (cd $dir && terraform init) + (cd $dir && terraform fmt -check) + (cd $dir && terraform validate) + (cd $dir && terraform plan -input=false) + done + + gcp_test: + name: gcp_test + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@master + + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - name: Install Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.3.4 + + - name: Loop through subdirectories + run: | + for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "gcp*"); do + (cd $dir && terraform init) + (cd $dir && terraform fmt -check) + (cd $dir && terraform validate) + (cd $dir && terraform plan -input=false) + done + + azure_test: + name: azure-test + runs-on: ubuntu-latest + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + steps: + - name: Checkout + uses: actions/checkout@v2 - name: Install Terraform uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.3.4 + - name: Loop through subdirectories + run: | + for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "azure*"); do + (cd $dir && terraform init) + (cd $dir && terraform fmt -check) + (cd $dir && terraform validate) + (cd $dir && terraform plan -input=false) + done + + deploy_test-runner-vm-to-azure: + uses: ./.github/workflows/deploy-test-runner.yml + secrets: inherit + + k3d_test: + name: k3d_test + needs: ["deploy_test-runner-vm-to-azure"] + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Loop through subdirectories run: | for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "k3d*"); do diff --git a/infrastructure/deploy.tf b/infrastructure/deploy.tf new file mode 100644 index 00000000..35a3ca24 --- /dev/null +++ b/infrastructure/deploy.tf @@ -0,0 +1,98 @@ +resource "azurerm_resource_group" "example" { + name = "zenml-test" + location = "West Europe" +} + +resource "azurerm_virtual_network" "example" { + name = "mlstack-test-network" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.example.location + resource_group_name = azurerm_resource_group.example.name +} + +resource "azurerm_subnet" "example" { + name = "mlstack-subnet" + resource_group_name = azurerm_resource_group.example.name + virtual_network_name = azurerm_virtual_network.example.name + address_prefixes = ["10.0.2.0/24"] +} + +resource "azurerm_network_interface" "example" { + name = "mlstack-nic" + location = azurerm_resource_group.example.location + resource_group_name = azurerm_resource_group.example.name + + ip_configuration { + name = "mlstack-ip" + subnet_id = azurerm_subnet.example.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.example.id + } +} + +resource "azurerm_public_ip" "example" { + name = "mlstack-pip" + location = azurerm_resource_group.example.location + resource_group_name = azurerm_resource_group.example.name + allocation_method = "Dynamic" +} + +resource "azurerm_network_security_group" "example" { + name = "mlstack-nsg" + location = azurerm_resource_group.example.location + resource_group_name = azurerm_resource_group.example.name +} + +resource "azurerm_network_security_rule" "example" { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = azurerm_resource_group.example.name + network_security_group_name = azurerm_network_security_group.example.name +} + +resource "azurerm_network_interface_security_group_association" "example" { + network_interface_id = azurerm_network_interface.example.id + network_security_group_id = azurerm_network_security_group.example.id +} + +data "azurerm_ssh_public_key" "example" { + name = "mlstack-test-vm" + resource_group_name = "zenml-developers" +} + +data "azurerm_image" "example" { + name = "mlstack-test-machine-image-20230818085527" + resource_group_name = "zenml-developers" +} + +resource "azurerm_linux_virtual_machine" "example" { + name = "mlstack-test-machine" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + size = "Standard_D8s_v3" + admin_username = "mlstackuser" + network_interface_ids = [ + azurerm_network_interface.example.id, + ] + + admin_ssh_key { + username = "mlstackuser" + public_key = data.azurerm_ssh_public_key.example.public_key + } + + os_disk { + caching = "ReadWrite" + storage_account_type = "StandardSSD_LRS" + } + + source_image_id = data.azurerm_image.example.id + + custom_data = base64encode("cd actions-runner\n./config.sh --url https://github.com/safoinme/mlops-stacks --token ${var.github_runner_token}\n./run.sh") +} \ No newline at end of file diff --git a/infrastructure/terraform.tf b/infrastructure/terraform.tf new file mode 100644 index 00000000..28831d4f --- /dev/null +++ b/infrastructure/terraform.tf @@ -0,0 +1,15 @@ +# defining the providers for the recipe module +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">=3.16.0" + } + } + + required_version = ">= 0.14.8" +} + +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf new file mode 100644 index 00000000..ffad7aeb --- /dev/null +++ b/infrastructure/variables.tf @@ -0,0 +1,4 @@ +variable "github_runner_token" { + description = "GitHub token" + type = string +} \ No newline at end of file From 2b4b2bc13361855b56403fe2abb70205e69fc5ee Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 12:27:42 +0100 Subject: [PATCH 06/20] add destroy --- .github/workflows/recipe-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index cf04a9ee..9787024d 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -9,6 +9,7 @@ on: branches: - main - develop + - feature/create-self-hosted-runner permissions: id-token: write @@ -130,3 +131,8 @@ jobs: (cd $dir && terraform validate) (cd $dir && terraform plan -input=false) done + + destroy_test-runner-vm-to-azure: + uses: ./.github/workflows/destroy-test-runner.yml + needs: ["deploy_test-runner-vm-to-azure", "k3d_test"] + secrets: inherit \ No newline at end of file From 71919d566e6a2de2b43e89b4fedeb20d8bf76c52 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 12:35:23 +0100 Subject: [PATCH 07/20] fix repo url --- infrastructure/deploy.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/deploy.tf b/infrastructure/deploy.tf index 35a3ca24..a29aa694 100644 --- a/infrastructure/deploy.tf +++ b/infrastructure/deploy.tf @@ -94,5 +94,5 @@ resource "azurerm_linux_virtual_machine" "example" { source_image_id = data.azurerm_image.example.id - custom_data = base64encode("cd actions-runner\n./config.sh --url https://github.com/safoinme/mlops-stacks --token ${var.github_runner_token}\n./run.sh") + custom_data = base64encode("cd actions-runner\n./config.sh --url https://github.com/zenml-io/mlops-stacks --token ${var.github_runner_token}\n./run.sh") } \ No newline at end of file From 329bb10615e44daf6de3a0aea11661911f297439 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 18 Aug 2023 13:19:55 +0100 Subject: [PATCH 08/20] add terraform backend to store the state --- infrastructure/deploy.tf | 2 +- infrastructure/terraform.tf | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/infrastructure/deploy.tf b/infrastructure/deploy.tf index a29aa694..fe8b04d1 100644 --- a/infrastructure/deploy.tf +++ b/infrastructure/deploy.tf @@ -94,5 +94,5 @@ resource "azurerm_linux_virtual_machine" "example" { source_image_id = data.azurerm_image.example.id - custom_data = base64encode("cd actions-runner\n./config.sh --url https://github.com/zenml-io/mlops-stacks --token ${var.github_runner_token}\n./run.sh") + custom_data = base64encode("cd actions-runner\n./config.sh --url https://github.com/zenml-io/mlops-stacks --token ${var.github_runner_token}\n./run.sh\necho 'OK' >> ./script_success.txt") } \ No newline at end of file diff --git a/infrastructure/terraform.tf b/infrastructure/terraform.tf index 28831d4f..b90ff680 100644 --- a/infrastructure/terraform.tf +++ b/infrastructure/terraform.tf @@ -8,6 +8,13 @@ terraform { } required_version = ">= 0.14.8" + + backend "azurerm" { + resource_group_name = "zenml-developers" + storage_account_name = "zenmlstorageaccount" + container_name = "github-runner-tf" + key = "terraform.tfstate" + } } provider "azurerm" { From 28f6d64115844449515e6c1aa78a5af1f88cba4c Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sat, 19 Aug 2023 16:38:21 +0100 Subject: [PATCH 09/20] change image --- .github/workflows/recipe-test.yml | 1 + infrastructure/deploy.tf | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index 9787024d..8e111a1d 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -132,6 +132,7 @@ jobs: (cd $dir && terraform plan -input=false) done + destroy_test-runner-vm-to-azure: uses: ./.github/workflows/destroy-test-runner.yml needs: ["deploy_test-runner-vm-to-azure", "k3d_test"] diff --git a/infrastructure/deploy.tf b/infrastructure/deploy.tf index fe8b04d1..40f4b5e9 100644 --- a/infrastructure/deploy.tf +++ b/infrastructure/deploy.tf @@ -68,7 +68,7 @@ data "azurerm_ssh_public_key" "example" { } data "azurerm_image" "example" { - name = "mlstack-test-machine-image-20230818085527" + name = "mlstack-github-runner-machine-image-20230819162059" resource_group_name = "zenml-developers" } @@ -93,6 +93,4 @@ resource "azurerm_linux_virtual_machine" "example" { } source_image_id = data.azurerm_image.example.id - - custom_data = base64encode("cd actions-runner\n./config.sh --url https://github.com/zenml-io/mlops-stacks --token ${var.github_runner_token}\n./run.sh\necho 'OK' >> ./script_success.txt") } \ No newline at end of file From 4c9efb887deb65b90702839c2babe0f17a76ff95 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Wed, 23 Aug 2023 12:41:15 +0100 Subject: [PATCH 10/20] return k3d-test to default runner --- .github/workflows/recipe-test.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/recipe-test.yml b/.github/workflows/recipe-test.yml index 8e111a1d..13fd59d7 100644 --- a/.github/workflows/recipe-test.yml +++ b/.github/workflows/recipe-test.yml @@ -111,18 +111,26 @@ jobs: (cd $dir && terraform plan -input=false) done - deploy_test-runner-vm-to-azure: - uses: ./.github/workflows/deploy-test-runner.yml - secrets: inherit + #deploy_test-runner-vm-to-azure: + # uses: ./.github/workflows/deploy-test-runner.yml + # secrets: inherit k3d_test: name: k3d_test - needs: ["deploy_test-runner-vm-to-azure"] - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 + - name: Install k3d + run: | + curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + + - name: Install Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.3.4 + - name: Loop through subdirectories run: | for dir in $(ls -d */ | grep -Fv -e "modules" -e ".github" | grep "k3d*"); do @@ -133,7 +141,7 @@ jobs: done - destroy_test-runner-vm-to-azure: - uses: ./.github/workflows/destroy-test-runner.yml - needs: ["deploy_test-runner-vm-to-azure", "k3d_test"] - secrets: inherit \ No newline at end of file + #destroy_test-runner-vm-to-azure: + # uses: ./.github/workflows/destroy-test-runner.yml + # needs: ["deploy_test-runner-vm-to-azure", "k3d_test"] + # secrets: inherit \ No newline at end of file From f3377e61a000c838ad521d7cae1c057b6d5c7a05 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Thu, 24 Aug 2023 09:50:03 +0100 Subject: [PATCH 11/20] Update infrastructure/terraform.tf Co-authored-by: Alex Strick van Linschoten --- infrastructure/terraform.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/infrastructure/terraform.tf b/infrastructure/terraform.tf index b90ff680..08650453 100644 --- a/infrastructure/terraform.tf +++ b/infrastructure/terraform.tf @@ -19,4 +19,13 @@ terraform { provider "azurerm" { features {} + default_tags { + tags = { + z-env = "dev" + z-owner = "safoine-ext" + z-project = "testing" + z-team = "oss" + z-description = "resources for integration testing" + } + } } \ No newline at end of file From 6abcbb115f3f63ba78dcfef36348a1278f4352fd Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Thu, 24 Aug 2023 09:54:10 +0100 Subject: [PATCH 12/20] apply suggested reviews --- .github/workflows/deploy-test-runner.yml | 9 +++++++-- infrastructure/deploy.tf | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-test-runner.yml b/.github/workflows/deploy-test-runner.yml index c9533ea3..2e014467 100644 --- a/.github/workflows/deploy-test-runner.yml +++ b/.github/workflows/deploy-test-runner.yml @@ -1,8 +1,13 @@ name: Deploy the test runner vm to azure on: - workflow_call: - workflow_dispatch: + workflow_call: + workflow_dispatch: + push: + branches: + - main + - develop + paths-ignore: ["**.md"] jobs: deploy_test_vm: diff --git a/infrastructure/deploy.tf b/infrastructure/deploy.tf index 40f4b5e9..67399cad 100644 --- a/infrastructure/deploy.tf +++ b/infrastructure/deploy.tf @@ -1,5 +1,5 @@ resource "azurerm_resource_group" "example" { - name = "zenml-test" + name = "zenml-github-test" location = "West Europe" } From 7cdb910eb511be8bea21e9088dab0bf6011c994b Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:11:36 +0100 Subject: [PATCH 13/20] Apply suggestions from code review Co-authored-by: Alex Strick van Linschoten --- .github/workflows/deploy-test-runner.yml | 1 + .github/workflows/destroy-test-runner.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/deploy-test-runner.yml b/.github/workflows/deploy-test-runner.yml index 2e014467..0100d5b1 100644 --- a/.github/workflows/deploy-test-runner.yml +++ b/.github/workflows/deploy-test-runner.yml @@ -18,6 +18,7 @@ jobs: ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + MLSTACKS_ANALYTICS_OPT_IN=false permissions: contents: "read" id-token: "write" diff --git a/.github/workflows/destroy-test-runner.yml b/.github/workflows/destroy-test-runner.yml index ee5fa2e2..80cf0e4a 100644 --- a/.github/workflows/destroy-test-runner.yml +++ b/.github/workflows/destroy-test-runner.yml @@ -13,6 +13,7 @@ jobs: ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + MLSTACKS_ANALYTICS_OPT_IN=false permissions: contents: "read" id-token: "write" From 3184c72923030497e32d340cd68ecafb8ac97eba Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 17:39:57 +0100 Subject: [PATCH 14/20] try new workflow to run on self-hosted runner --- .github/workflows/ci.yml | 15 ++++++++++++++- ...-runner.yml => deploy-self-hosted-runner.yml} | 15 ++++++++++++++- ...runner.yml => destroy-self-hosted-runner.yml} | 16 ++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) rename .github/workflows/{deploy-test-runner.yml => deploy-self-hosted-runner.yml} (69%) rename .github/workflows/{destroy-test-runner.yml => destroy-self-hosted-runner.yml} (65%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 587dc751..a4d10e4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,9 +194,16 @@ jobs: (cd $dir && terraform plan -input=false) done + provion-self-hosted-runner: + name: provion-self-hosted-runner + uses: ./.github/workflows/deploy-self-hosted-runner.yml + secrets: inherit + k3d_test: name: k3d_test - runs-on: ubuntu-latest + runs-on: self-hosted + needs: + - provion-self-hosted-runner steps: - name: Checkout uses: actions/checkout@v2 @@ -218,3 +225,9 @@ jobs: (cd $dir && terraform validate) (cd $dir && terraform plan -input=false) done + + destory-self-hosted-runner: + name: destory-self-hosted-runner + needs: k3d_test + if: always() + uses: ./.github/workflows/destroy-self-hosted-runner.yml \ No newline at end of file diff --git a/.github/workflows/deploy-test-runner.yml b/.github/workflows/deploy-self-hosted-runner.yml similarity index 69% rename from .github/workflows/deploy-test-runner.yml rename to .github/workflows/deploy-self-hosted-runner.yml index 2e014467..5a4cf74a 100644 --- a/.github/workflows/deploy-test-runner.yml +++ b/.github/workflows/deploy-self-hosted-runner.yml @@ -30,6 +30,14 @@ jobs: - name: Checkout the Code uses: actions/checkout@v3 + - name: Install Azure CLI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - name: Login to Azure + run: | + az login --service-principal --username $ARM_CLIENT_ID --password $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID + - name: Setup Terraform uses: hashicorp/setup-terraform@v2 @@ -48,4 +56,9 @@ jobs: - run: terraform apply -auto-approve env: - TF_VAR_github_runner_token: ${{ secrets.runner_token }} \ No newline at end of file + TF_VAR_github_runner_token: ${{ secrets.runner_token }} + + - name: Create blob + run: | + echo "Creating blob..." + echo "" | az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block \ No newline at end of file diff --git a/.github/workflows/destroy-test-runner.yml b/.github/workflows/destroy-self-hosted-runner.yml similarity index 65% rename from .github/workflows/destroy-test-runner.yml rename to .github/workflows/destroy-self-hosted-runner.yml index ee5fa2e2..1eefc24b 100644 --- a/.github/workflows/destroy-test-runner.yml +++ b/.github/workflows/destroy-self-hosted-runner.yml @@ -41,6 +41,18 @@ jobs: id: validate run: terraform validate -no-color - - run: terraform destroy -auto-approve -refresh=False + - name: Delete blob + run: | + az storage blob delete --account-name zenmlstorageaccount --container-name github-runner-tf --name ${{ github.run_id }} + + - name: Check if any blobs left + id: check_blobs + run: | + blobs=$(az storage blob list --account-name zenmlstorageaccount --container-name github-runner-tf --prefix github-run --output tsv) + echo "BLOBS=$blobs" >> $GITHUB_ENV + + - name: Destroy VM + run: terraform destroy -auto-approve -refresh=False env: - TF_VAR_github_runner_token: ${{ secrets.runner_token }} \ No newline at end of file + TF_VAR_github_runner_token: ${{ secrets.runner_token }} + if: env.BLOBS == '' \ No newline at end of file From e77ac3a4bc19c7eb1e2a2b7ebf9672a45df60b00 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 17:45:43 +0100 Subject: [PATCH 15/20] format --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d1bbb02..a5a11bdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: k3d_test: name: k3d_test runs-on: self-hosted - needs: + needs: - provion-self-hosted-runner steps: - name: Checkout @@ -229,4 +229,4 @@ jobs: name: destory-self-hosted-runner needs: k3d_test if: always() - uses: ./.github/workflows/destroy-self-hosted-runner.yml \ No newline at end of file + uses: ./.github/workflows/destroy-self-hosted-runner.yml From fc460125fb7f1b2acc23476ceb44f960acc7bcd9 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 17:49:08 +0100 Subject: [PATCH 16/20] fix destory yml --- .github/workflows/destroy-self-hosted-runner.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/destroy-self-hosted-runner.yml b/.github/workflows/destroy-self-hosted-runner.yml index f56fb174..1eefc24b 100644 --- a/.github/workflows/destroy-self-hosted-runner.yml +++ b/.github/workflows/destroy-self-hosted-runner.yml @@ -13,7 +13,6 @@ jobs: ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} - MLSTACKS_ANALYTICS_OPT_IN=false permissions: contents: "read" id-token: "write" From f88cf996cc30057b9bbd0b1ad944cd5d189f1ac1 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 17:50:12 +0100 Subject: [PATCH 17/20] fix deploy yml --- .github/workflows/deploy-self-hosted-runner.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-self-hosted-runner.yml b/.github/workflows/deploy-self-hosted-runner.yml index 43fc09c1..5a4cf74a 100644 --- a/.github/workflows/deploy-self-hosted-runner.yml +++ b/.github/workflows/deploy-self-hosted-runner.yml @@ -18,7 +18,6 @@ jobs: ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} - MLSTACKS_ANALYTICS_OPT_IN=false permissions: contents: "read" id-token: "write" From 7d3004960f133cde2320c466bc7ba2da1d45d365 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 17:53:07 +0100 Subject: [PATCH 18/20] add tags to resource groups --- infrastructure/deploy.tf | 8 ++++++++ infrastructure/terraform.tf | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/infrastructure/deploy.tf b/infrastructure/deploy.tf index 67399cad..8af1c22a 100644 --- a/infrastructure/deploy.tf +++ b/infrastructure/deploy.tf @@ -1,6 +1,14 @@ resource "azurerm_resource_group" "example" { name = "zenml-github-test" location = "West Europe" + + tags = { + z-env = "dev" + z-owner = "safoine-ext" + z-project = "testing" + z-team = "oss" + z-description = "resources for integration testing" + } } resource "azurerm_virtual_network" "example" { diff --git a/infrastructure/terraform.tf b/infrastructure/terraform.tf index 08650453..b90ff680 100644 --- a/infrastructure/terraform.tf +++ b/infrastructure/terraform.tf @@ -19,13 +19,4 @@ terraform { provider "azurerm" { features {} - default_tags { - tags = { - z-env = "dev" - z-owner = "safoine-ext" - z-project = "testing" - z-team = "oss" - z-description = "resources for integration testing" - } - } } \ No newline at end of file From 83734bae0cc37db421660be825b3fbd803f7b1c5 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 18:00:42 +0100 Subject: [PATCH 19/20] update blob write and check --- .github/workflows/deploy-self-hosted-runner.yml | 2 +- .github/workflows/destroy-self-hosted-runner.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-self-hosted-runner.yml b/.github/workflows/deploy-self-hosted-runner.yml index 5a4cf74a..baae536f 100644 --- a/.github/workflows/deploy-self-hosted-runner.yml +++ b/.github/workflows/deploy-self-hosted-runner.yml @@ -61,4 +61,4 @@ jobs: - name: Create blob run: | echo "Creating blob..." - echo "" | az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block \ No newline at end of file + echo "" | az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block --data "" \ No newline at end of file diff --git a/.github/workflows/destroy-self-hosted-runner.yml b/.github/workflows/destroy-self-hosted-runner.yml index 1eefc24b..ca47ac64 100644 --- a/.github/workflows/destroy-self-hosted-runner.yml +++ b/.github/workflows/destroy-self-hosted-runner.yml @@ -48,7 +48,7 @@ jobs: - name: Check if any blobs left id: check_blobs run: | - blobs=$(az storage blob list --account-name zenmlstorageaccount --container-name github-runner-tf --prefix github-run --output tsv) + blobs=$(az storage blob list --account-name zenmlstorageaccount --container-name github-runner-tf --query "[?starts_with(name, 'github-run')].name" --output tsv) echo "BLOBS=$blobs" >> $GITHUB_ENV - name: Destroy VM From 0e8d71d6073dc469fcddf4f70311cf64400891f7 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Sun, 24 Sep 2023 18:05:33 +0100 Subject: [PATCH 20/20] update blob write and check --- .github/workflows/deploy-self-hosted-runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-self-hosted-runner.yml b/.github/workflows/deploy-self-hosted-runner.yml index baae536f..0e2e8ed2 100644 --- a/.github/workflows/deploy-self-hosted-runner.yml +++ b/.github/workflows/deploy-self-hosted-runner.yml @@ -61,4 +61,4 @@ jobs: - name: Create blob run: | echo "Creating blob..." - echo "" | az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block --data "" \ No newline at end of file + az storage blob upload --account-name zenmlstorageaccount --container-name github-runner-tf --name github-run-${{ github.run_id }} --type block --data "${{ github.run_id }}" \ No newline at end of file