feat: Migrate E2E pipeline to using Self-Hosted Runner (#638)

**Reason for Change**: To remove dependency on GitHub secrets and environments (which are only accessible by maintainers). Use Self-Hosted runner. --------- Co-authored-by: Fei Guo <[email protected]>
kaito-project · Oct 18, 2024 · b6694c2 · b6694c2
1 parent 7a9468c
commit b6694c2
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 29 deletions.
diff --git a/.github/workflows/e2e-workflow.yml b/.github/workflows/e2e-workflow.yml
@@ -39,7 +39,7 @@ on:
 
 jobs:
   e2e-tests:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, 'hostname:kaito-e2e-github-runner']
     name: e2e-tests-${{ inputs.node_provisioner }}
     permissions:
       contents: read
@@ -49,6 +49,13 @@ jobs:
       GO_VERSION: "1.22"
       KARPENTER_NAMESPACE: "karpenter"
       GPU_PROVISIONER_NAMESPACE: "gpu-provisioner"
+      E2E_CLIENT_ID: $E2E_CLIENT_ID
+      E2E_TENANT_ID: $E2E_TENANT_ID
+      E2E_SUBSCRIPTION_ID: $E2E_SUBSCRIPTION_ID
+      E2E_AMRT_SECRET_NAME: $E2E_AMRT_SECRET_NAME
+      E2E_ACR_AMRT_USERNAME: $E2E_ACR_AMRT_USERNAME
+      E2E_ACR_AMRT_PASSWORD: $E2E_ACR_AMRT_PASSWORD
+
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
@@ -87,13 +94,13 @@ jobs:
       - name: Az login
         uses: azure/[email protected]
         with:
-          client-id: ${{ secrets.E2E_CLIENT_ID }}
-          tenant-id: ${{ secrets.E2E_TENANT_ID }}
-          subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          client-id: ${{ env.E2E_CLIENT_ID }}
+          tenant-id: ${{ env.E2E_TENANT_ID }}
+          subscription-id: ${{ env.E2E_SUBSCRIPTION_ID }}
 
       - uses: azure/setup-helm@v4
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ env.GITHUB_TOKEN }}
         id: install
 
       - name: Create Resource Group
@@ -162,9 +169,9 @@ jobs:
       - name: Az login
         uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
         with:
-          client-id: ${{ secrets.E2E_CLIENT_ID }}
-          tenant-id: ${{ secrets.E2E_TENANT_ID }}
-          subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          client-id: ${{ env.E2E_CLIENT_ID }}
+          tenant-id: ${{ env.E2E_TENANT_ID }}
+          subscription-id: ${{ env.E2E_SUBSCRIPTION_ID }}
 
       - name: Create Identities and Permissions for ${{ inputs.node_provisioner }}
         shell: bash
@@ -174,7 +181,7 @@ jobs:
           AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
           AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
           TEST_SUITE: ${{ inputs.node_provisioner }}
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ env.E2E_SUBSCRIPTION_ID }}
 
       - name: Install gpu-provisioner helm chart
         if: ${{ inputs.node_provisioner == 'gpuprovisioner' }}
@@ -184,8 +191,8 @@ jobs:
         env:
           AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
           AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
-          AZURE_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          AZURE_TENANT_ID: ${{ env.E2E_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ env.E2E_SUBSCRIPTION_ID }}
           GPU_PROVISIONER_VERSION: ${{ vars.GPU_PROVISIONER_VERSION }}
 
       - name: Install karpenter Azure provider helm chart
@@ -196,16 +203,16 @@ jobs:
         env:
           AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
           AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
-          AZURE_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          AZURE_TENANT_ID: ${{ env.E2E_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ env.E2E_SUBSCRIPTION_ID }}
           KARPENTER_VERSION: ${{ vars.KARPENTER_VERSION }}
           KARPENTER_NAMESPACE: ${{ env.KARPENTER_NAMESPACE }}
 
       - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
         with:
-          client-id: ${{ secrets.E2E_CLIENT_ID }}
-          tenant-id: ${{ secrets.E2E_TENANT_ID }}
-          subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}
+          client-id: ${{ env.E2E_CLIENT_ID }}
+          tenant-id: ${{ env.E2E_TENANT_ID }}
+          subscription-id: ${{ env.E2E_SUBSCRIPTION_ID }}
 
       - name: build KAITO image
         if: ${{ !inputs.isRelease }}
@@ -251,10 +258,10 @@ jobs:
       # Add Private-Hosted ACR secret for private models like llama
       - name: Add Private-Hosted ACR Secret Credentials
         run: |
-          kubectl create secret docker-registry ${{ secrets.E2E_AMRT_SECRET_NAME }} \
-          --docker-server=${{ secrets.E2E_ACR_AMRT_USERNAME }}.azurecr.io \
-          --docker-username=${{ secrets.E2E_ACR_AMRT_USERNAME }} \
-          --docker-password=${{ secrets.E2E_ACR_AMRT_PASSWORD }}
+          kubectl create secret docker-registry ${{ env.E2E_AMRT_SECRET_NAME }} \
+          --docker-server=${{ env.E2E_ACR_AMRT_USERNAME }}.azurecr.io \
+          --docker-username=${{ env.E2E_ACR_AMRT_USERNAME }} \
+          --docker-password=${{ env.E2E_ACR_AMRT_PASSWORD }}
 
       - name: Log ${{ inputs.node_provisioner }}
         run: |
@@ -275,8 +282,8 @@ jobs:
           AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
           RUN_LLAMA_13B: ${{ env.RUN_LLAMA_13B }}
           REGISTRY: ${{ env.REGISTRY }}
-          AI_MODELS_REGISTRY: ${{ secrets.E2E_ACR_AMRT_USERNAME }}.azurecr.io
-          AI_MODELS_REGISTRY_SECRET: ${{ secrets.E2E_AMRT_SECRET_NAME }}
+          AI_MODELS_REGISTRY: ${{ env.E2E_ACR_AMRT_USERNAME }}.azurecr.io
+          AI_MODELS_REGISTRY_SECRET: ${{ env.E2E_AMRT_SECRET_NAME }}
           TEST_SUITE: ${{ inputs.node_provisioner }}
           E2E_ACR_REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
           E2E_ACR_REGISTRY_SECRET: ${{ env.CLUSTER_NAME }}-acr-secret

diff --git a/.github/workflows/kaito-e2e.yml b/.github/workflows/kaito-e2e.yml
@@ -30,10 +30,3 @@ jobs:
       git_sha: ${{ github.event.pull_request.head.sha }}
       k8s_version: ${{ vars.AKS_K8S_VERSION }}
       node_provisioner: ${{ matrix.node-provisioner }}
-    secrets:
-      E2E_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
-      E2E_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-      E2E_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      E2E_AMRT_SECRET_NAME: ${{ secrets.AMRT_SECRET_NAME }}
-      E2E_ACR_AMRT_USERNAME: ${{ secrets.ACR_AMRT_USERNAME }}
-      E2E_ACR_AMRT_PASSWORD: ${{ secrets.ACR_AMRT_PASSWORD }}