e2e-workflow.yml

name: kaito-e2e-workflow

on:
  workflow_call:
    inputs:
      git_sha:
        type: string
        required: true
      node_provisioner:
        type: string
        required: false
        default: gpuprovisioner
      tag:
        type: string
      isRelease:
        type: boolean
        default: false
      registry:
        type: string
      region:
        type: string
        description: "the azure location to run the e2e test in"
        default: "eastus"
      k8s_version:
        type: string
    secrets:
      E2E_CLIENT_ID:
        required: true
      E2E_TENANT_ID:
        required: true
      E2E_SUBSCRIPTION_ID:
        required: true
      E2E_AMRT_SECRET_NAME:
        required: true
      E2E_ACR_AMRT_USERNAME:
        required: true
      E2E_ACR_AMRT_PASSWORD:
        required: true

jobs:
  e2e-tests:
    runs-on: ubuntu-latest
    name: e2e-tests-${{ inputs.node_provisioner }}
    permissions:
      contents: read
      id-token: write # This is required for requesting the JWT
    environment: e2e-test
    env:
      GO_VERSION: "1.22"
      KARPENTER_NAMESPACE: "karpenter"
      GPU_PROVISIONER_NAMESPACE: "gpu-provisioner"
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
        with:
          egress-policy: audit

      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.git_sha }}

      - name: Set e2e Resource and Cluster Name
        run: |
          rand=$(git rev-parse --short ${{ inputs.git_sha }})

          if [ "$rand" = "" ]; then
             rand=$RANDOM
          fi

          echo "VERSION=${rand}" >> $GITHUB_ENV
          echo "CLUSTER_NAME=${{ inputs.node_provisioner }}${rand}" >> $GITHUB_ENV
          echo "REGISTRY=${{ inputs.node_provisioner }}${rand}.azurecr.io" >> $GITHUB_ENV
          echo "RUN_LLAMA_13B=false" >> $GITHUB_ENV

      - name: Set Registry
        if: ${{ inputs.isRelease }}
        run: |
          echo "REGISTRY=${{ inputs.registry }}" >> $GITHUB_ENV
          echo "VERSION=$(echo ${{ inputs.tag }} | tr -d v)" >> $GITHUB_ENV

      - name: Set up Go ${{ env.GO_VERSION }}
        uses: actions/setup-go@v5.0.2
        with:
          go-version: ${{ env.GO_VERSION  }}

      - name: Az login
        uses: azure/login@v2.2.0
        with:
          client-id: ${{ secrets.E2E_CLIENT_ID }}
          tenant-id: ${{ secrets.E2E_TENANT_ID }}
          subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}

      - uses: azure/setup-helm@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
        id: install

      - name: Create Resource Group
        shell: bash
        run: |
          make create-rg
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}

      - name: Create ACR
        shell: bash
        run: |
          make create-acr
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}

      - name: Create Azure Identity
        uses: azure/CLI@v2.1.0
        with:
          inlineScript: |
            az identity create --name ${{ inputs.node_provisioner }}Identity --resource-group  ${{ env.CLUSTER_NAME }}

      - name: Generate APIs
        run: |
          make generate

      - name: build KAITO image
        if: ${{ !inputs.isRelease }}
        shell: bash
        run: |
          make docker-build-workspace
        env:
          REGISTRY: ${{ env.REGISTRY }}
          VERSION: ${{ env.VERSION }}

      - name: build adapter image
        shell: bash
        run: |
          make docker-build-adapter
        env:
          REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io

      - name: build dataset image
        shell: bash
        run: |
          make docker-build-dataset
        env:
          REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io

      - name: create cluster
        shell: bash
        run: |
          if [ "${{ inputs.node_provisioner }}" == "gpuprovisioner" ]; then
             make create-aks-cluster
          else
            make create-aks-cluster-for-karpenter
          fi
        env:
          AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          AZURE_LOCATION: ${{ inputs.region }}
          AKS_K8S_VERSION: ${{ inputs.k8s_version }}

      - name: Az login
        uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
        with:
          client-id: ${{ secrets.E2E_CLIENT_ID }}
          tenant-id: ${{ secrets.E2E_TENANT_ID }}
          subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}

      - name: Create Identities and Permissions for ${{ inputs.node_provisioner }}
        shell: bash
        run: |
          make generate-identities
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          TEST_SUITE: ${{ inputs.node_provisioner }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}

      - name: Install gpu-provisioner helm chart
        if: ${{ inputs.node_provisioner == 'gpuprovisioner' }}
        shell: bash
        run: |
          make gpu-provisioner-helm
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          AZURE_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
          GPU_PROVISIONER_VERSION: ${{ vars.GPU_PROVISIONER_VERSION }}

      - name: Install karpenter Azure provider helm chart
        if: ${{ inputs.node_provisioner == 'azkarpenter' }}
        shell: bash
        run: |
          make azure-karpenter-helm
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          AZURE_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
          KARPENTER_VERSION: ${{ vars.KARPENTER_VERSION }}
          KARPENTER_NAMESPACE: ${{ env.KARPENTER_NAMESPACE }}

      - uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
        with:
          client-id: ${{ secrets.E2E_CLIENT_ID }}
          tenant-id: ${{ secrets.E2E_TENANT_ID }}
          subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}

      - name: build KAITO image
        if: ${{ !inputs.isRelease }}
        shell: bash
        run: |
          make docker-build-workspace
        env:
          REGISTRY: ${{ env.REGISTRY }}
          VERSION: ${{ env.VERSION }}

      - name: Install KAITO Workspace helm chart
        shell: bash
        run: |
          make az-patch-install-helm
          kubectl wait --for=condition=available deploy "kaito-workspace" -n kaito-workspace --timeout=300s
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          REGISTRY: ${{ env.REGISTRY }}
          VERSION: ${{ env.VERSION }}
          TEST_SUITE: ${{ inputs.node_provisioner }}

      # Retrieve E2E ACR credentials and create Kubernetes secret
      - name: Set up E2E ACR Credentials and Secret
        shell: bash
        run: |
          # Retrieve the ACR username and password
          ACR_USERNAME=$(az acr credential show --name ${{ env.CLUSTER_NAME }} --resource-group ${{ env.CLUSTER_NAME }} --query "username" -o tsv)
          ACR_PASSWORD=$(az acr credential show --name ${{ env.CLUSTER_NAME }} --resource-group ${{ env.CLUSTER_NAME }} --query "passwords[0].value" -o tsv)

          # Ensure credentials were retrieved successfully
          if [ -z "$ACR_USERNAME" ] || [ -z "$ACR_PASSWORD" ]; then
            echo "Failed to retrieve ACR credentials"
            exit 1
          fi

          # Create the Kubernetes secret with the retrieved credentials
          kubectl create secret docker-registry ${{ env.CLUSTER_NAME }}-acr-secret \
          --docker-server=${{ env.CLUSTER_NAME }}.azurecr.io \
          --docker-username=${ACR_USERNAME} \
          --docker-password=${ACR_PASSWORD}

      # Add Private-Hosted ACR secret for private models like llama
      - name: Add Private-Hosted ACR Secret Credentials
        run: |
          kubectl create secret docker-registry ${{ secrets.E2E_AMRT_SECRET_NAME }} \
          --docker-server=${{ secrets.E2E_ACR_AMRT_USERNAME }}.azurecr.io \
          --docker-username=${{ secrets.E2E_ACR_AMRT_USERNAME }} \
          --docker-password=${{ secrets.E2E_ACR_AMRT_PASSWORD }}

      - name: Log ${{ inputs.node_provisioner }}
        run: |
          if [ "${{ inputs.node_provisioner }}" == "gpuprovisioner" ]; then
            kubectl logs -n "${{ env.GPU_PROVISIONER_NAMESPACE }}" -l app.kubernetes.io/name=gpu-provisioner -c controller
          else
            kubectl logs -n "${{ env.KARPENTER_NAMESPACE }}" -l app.kubernetes.io/name=karpenter -c controller
          fi

      - name: Log kaito-workspace
        run: |
          kubectl get pods -n kaito-workspace -o name | grep "^pod/kaito-workspace" | sed 's/^pod\///' | xargs -I {} kubectl logs -n kaito-workspace {}

      - name: Run e2e test
        run: |
          make kaito-workspace-e2e-test
        env:
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          RUN_LLAMA_13B: ${{ env.RUN_LLAMA_13B }}
          REGISTRY: ${{ env.REGISTRY }}
          AI_MODELS_REGISTRY: ${{ secrets.E2E_ACR_AMRT_USERNAME }}.azurecr.io
          AI_MODELS_REGISTRY_SECRET: ${{ secrets.E2E_AMRT_SECRET_NAME }}
          TEST_SUITE: ${{ inputs.node_provisioner }}
          E2E_ACR_REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
          E2E_ACR_REGISTRY_SECRET: ${{ env.CLUSTER_NAME }}-acr-secret

      - name: Cleanup e2e resources
        if: ${{ always() }}
        uses: azure/CLI@v2.1.0
        with:
          inlineScript: |
            set +e
            az group delete --name "${{ env.CLUSTER_NAME }}" --yes --no-wait || true