terraform.yml

name: Workflow for deploying and destroying an ACR or AKS, as well as installing or uninstalling Helm charts on those Kubernetes clusters

on:
  workflow_dispatch:
    inputs:
      INFRASTRUCTURE_OPERATIONS:
        type: choice
        options: [ 'storage-account-backend-deploy', 'k8s-service-deploy', 'k8s-service-destroy', 'ml-ops-tools-install', 'ml-ops-tools-uninstall' ]
        default: k8s-service-deploy
        description: 'Infrastructure operations: [ storage-account-backend-deploy, k8s-service-deploy, k8s-service-destroy, ml-ops-tools-install, ml-ops-tools-uninstall ]'
      ML_OPS_TOOL:
        type: choice
        options: [ 'kubeflow', 'mlflow' ]
        default: 
        description: 'Select ML Ops tool: [ kubeflow, mlflow ]'
      ENVIRONMENT:
        type: choice
        options: [ 'sbx' ]
        default: sbx
        description: 'Environment on which to deploy: [ sbx ]. Dev, staging, prod environments not considered'
env:
  # AZ
  ARM_SUBSCRIPTION_ID: '${{ secrets.ARM_SUBSCRIPTION_ID }}'
  ARM_TENANT_ID: '${{ secrets.ARM_TENANT_ID }}'
  ARM_CLIENT_ID: '${{ secrets.ARM_CLIENT_ID }}'
  ARM_CLIENT_SECRET: '${{ secrets.ARM_CLIENT_SECRET }}'
  
jobs:
  deploy-tf-backend:
    runs-on: ubuntu-latest
    if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'storage-account-backend-deploy' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@master
      - name: Set up Terraform
        uses: hashicorp/setup-terraform@v3  
      - name: Terraform Init
        run: terraform init # should be only deployed once. Ensure to manually destroy the ACR in the Azure Portal Web UI
        working-directory: ./devops/terraform
      - name: Deploy Storage Account backend
        run: terraform apply --auto-approve
        continue-on-error: false
        working-directory: ./devops/terraform
  tf-k8s-service-operations:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@master
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-destroy' }}
      - name: Set up Terraform
        uses: hashicorp/setup-terraform@v3  
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-destroy' }}
      - name: Terraform Init # requires a Storage Account backend deployed trough storage-account-backend-deploy workflow step
        run: |
          terraform init \
          -backend-config="subscription_id=${{ env.ARM_SUBSCRIPTION_ID }}" \
          -backend-config="storage_account_name=gftfbesbxsa001" \
          -backend-config="resource_group_name=gftfbe-sbx-rg001" \
          -backend-config="container_name=gftfbesbxsac001" \
          -backend-config="key=sbx-k8s-service-deployment/terraform.tfstate"
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-deployment
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-destroy' }}
      - name: Deploy Kubernetes service and related resources
        run: terraform apply --auto-approve
        continue-on-error: false
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-deployment
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy' }}
      - name: Destroy Kubernetes service and related resources
        run: terraform destroy --auto-approve
        continue-on-error: false
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-deployment
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-destroy' }}
  tf-ml-ops-tools-operations:
    runs-on: ubuntu-latest
    needs: tf-k8s-service-operations
    if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'ml-ops-tools-install' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'ml-ops-tools-uninstall' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@master
      - name: Set up Terraform
        uses: hashicorp/setup-terraform@v3  
      - name: Download the ~/.kube/config
        shell: bash
        run: |
          terraform init \
          -backend-config="subscription_id=${{ env.ARM_SUBSCRIPTION_ID }}" \
          -backend-config="storage_account_name=gftfbesbxsa001" \
          -backend-config="resource_group_name=gftfbe-sbx-rg001" \
          -backend-config="container_name=gftfbesbxsac001" \
          -backend-config="key=sbx-k8s-service-deployment/terraform.tfstate"

          terraform output aks_kube_config_list | awk '/^  apiVersion:/,/^  EOT,$/' | sed 's/^  //' > ./config
          mkdir -vp ~/.kube
          head -n -3 ./config > ~/.kube/config
          cat ~/.kube/config
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-deployment
      - name: Install Kustomize and Kubeflow
        run: |
          sudo apt-get install -y git

          # Install kustomize
          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"  | bash

          # Install Kubeflow on Azure Kubernetes Service - https://github.com/Azure/kubeflow-aks
          cd /tmp/
          git clone --recurse-submodules https://github.com/Azure/kubeflow-aks.git
          cd kubeflow-aks/manifests/
          git checkout v1.8-branch
          cd ..
          cp -a deployments/vanilla manifests/vanilla
          cd manifests  
          while ! kustomize build vanilla | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 10; done
        
          kubectl get pods -n cert-manager
          kubectl get pods -n istio-system
          kubectl get pods -n auth
          kubectl get pods -n knative-eventing
          kubectl get pods -n knative-serving
          kubectl get pods -n kubeflow
          kubectl get pods -n kubeflow-user-example-com
        if: ${{ ((github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'ml-ops-tools-install' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy') && github.event.inputs.ML_OPS_TOOL == 'kubeflow') }}
      - name: Terraform Init # requires a Storage Account backend deployed trough storage-account-backend-deploy workflow step
        run: |
          terraform init \
          -backend-config="subscription_id=${{ env.ARM_SUBSCRIPTION_ID }}" \
          -backend-config="storage_account_name=gftfbesbxsa001" \
          -backend-config="resource_group_name=gftfbe-sbx-rg001" \
          -backend-config="container_name=gftfbesbxsac001" \
          -backend-config="key=sbx-k8s-service-configuration/terraform.tfstate"
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-configuration
      - name: Install helm charts
        shell: bash
        run: terraform apply --auto-approve
        continue-on-error: false
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-configuration
        env:
          TF_VAR_acr_username: "${{ secrets.ACR_USERNAME }}"
          TF_VAR_acr_password: "${{ secrets.ACR_PASSWORD }}"
          TF_VAR_acr_login_server_name: "${{ secrets.ACR_LOGIN_SERVER_NAME }}"
          TF_VAR_ml_ops_tool: "${{ github.event.inputs.ML_OPS_TOOL }}"
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'ml-ops-tools-install' || github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'k8s-service-deploy' }}
      - name: Uninstall helm charts
        run: terraform destroy --auto-approve
        continue-on-error: false
        working-directory: ./terraform/envs/${{ github.event.inputs.ENVIRONMENT }}-k8s-configuration
        if: ${{ github.event.inputs.INFRASTRUCTURE_OPERATIONS == 'ml-ops-tools-uninstall' }}