Run E2E tests on KOPS cluster (#27)

Update script to create/delete cluster automatically --------- Co-authored-by: Vladislav Volodkin <[email protected]>
awslabs · Nov 3, 2023 · 54bc3a1 · 54bc3a1
1 parent 566696a
commit 54bc3a1
Show file tree

Hide file tree

Showing 12 changed files with 446 additions and 71 deletions.
diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
@@ -18,14 +18,16 @@ on:
 concurrency: e2e-cluster
 env:
   AWS_REGION : "us-east-1"
-  KUBECONFIG: "/tmp/kubeconfig"
   COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }}
   TMP_IMAGE_NAME: "s3-csi-driver-tmp"
   PROMOTED_IMAGE_NAME: "s3-csi-driver"
 jobs:
   build:
     # this is to prevent the job to run at forked projects
     if: github.repository == 'awslabs/mountpoint-s3-csi-driver'
+    strategy:
+      matrix:
+        cluster-type: ["kops", "eksctl"]
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -60,18 +62,46 @@ jobs:
           export TAG=${{ env.COMMIT_ID }}
           make build_image
           make push_image
+      - name: Install tools
+        run: |
+          export ACTION=install_tools
+          tests/e2e-kubernetes/run.sh
+      - name: Create cluster
+        run: |
+          export ACTION=create_cluster
+          export AWS_REGION=${{ env.AWS_REGION }}
+          export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          tests/e2e-kubernetes/run.sh
       - name: Install the driver
-        env:
-          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }}
         run: |
-          export EKS_REGION=${{ env.AWS_REGION }}
-          export EKS_CLUSTER_NAME=s3-csi-cluster
-          export KUBECONFIG=${{ env.KUBECONFIG }}
+          export ACTION=install_driver
+          export AWS_REGION=${{ env.AWS_REGION }}
+          export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export IMAGE_NAME=${{ env.TMP_IMAGE_NAME }}
           export TAG=${{ env.COMMIT_ID }}
-          tests/e2e-kubernetes/install.sh
+          tests/e2e-kubernetes/run.sh
       - name: Run E2E Tests
-        run: make e2e E2E_KUBECONFIG=${{ env.KUBECONFIG }} E2E_COMMIT_ID=${{ env.COMMIT_ID }}
+        run: |
+          cd tests/e2e-kubernetes
+          export ACTION=run_tests
+          export AWS_REGION=${{ env.AWS_REGION }}
+          export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export TAG=${{ env.COMMIT_ID }}
+          ./run.sh
+      - name: Uinstall the driver
+        if: always()
+        run: |
+          export ACTION=uninstall_driver
+          export AWS_REGION=${{ env.AWS_REGION }}
+          export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          tests/e2e-kubernetes/run.sh
+      - name: Delete cluster
+        if: always()
+        run: |
+          export ACTION=delete_cluster
+          export AWS_REGION=${{ env.AWS_REGION }}
+          export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          tests/e2e-kubernetes/run.sh
       - name: Promote image for release branch
         if: ${{ startsWith(github.ref_name, 'release') }}
         env:

diff --git a/Makefile b/Makefile
@@ -64,6 +64,10 @@ test:
 	# skipping controller test cases because we don't implement controller for static provisioning, this is a known limitation of sanity testing package: https://github.com/kubernetes-csi/csi-test/issues/214
 	go test -v ./tests/sanity/... -ginkgo.skip="ControllerGetCapabilities" -ginkgo.skip="ValidateVolumeCapabilities"
 
+.PHONY: fmt
+fmt:
+	go fmt ./...
+
 .PHONY: e2e
 e2e:
 	pushd tests/e2e-kubernetes; \
@@ -72,10 +76,6 @@ e2e:
 	popd; \
 	exit $$EXIT_CODE
 
-.PHONY: fmt
-fmt:
-	go fmt ./...
-
 .PHONY: check_style
 check_style:
 	test -z "$$(gofmt -d . | tee /dev/stderr)"

diff --git a/examples/kubernetes/static_provisioning/static_provisioning.yaml b/examples/kubernetes/static_provisioning/static_provisioning.yaml
@@ -10,8 +10,6 @@ spec:
   mountOptions:
     - allow-delete
     - region eu-west-1
-    - uid 1001
-    - allow-other
   csi:
     driver: s3.csi.aws.com # required
     volumeHandle: s3-csi-driver # bucket name, required

diff --git a/tests/e2e-kubernetes/.gitignore b/tests/e2e-kubernetes/.gitignore
@@ -0,0 +1 @@
+csi-test-artifacts/*
diff --git a/tests/e2e-kubernetes/README.md b/tests/e2e-kubernetes/README.md
@@ -1,13 +1,52 @@
 
-## Usage
+# Usage
+## Prerequisites
+AWS credentials in ENVs with the following policies attached:
+```
+AmazonEC2FullAccess
+AmazonRoute53FullAccess
+AmazonS3FullAccess
+IAMFullAccess
+AmazonVPCFullAccess
+AmazonSQSFullAccess
+AmazonEventBridgeFullAccess
+AmazonSSMReadOnlyAccess
+```
+
+## Setting up the environment
+All of the following commands are expected to be executed from repo root:
+
+```bash
+ACTION=install_tools tests/e2e-kubernetes/run.sh
+
+ACTION=create_cluster AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh # set KOPS_STATE_FILE to your bucket when running locally
+
+ACTION=install_driver AWS_REGION=us-east-1 CLUSTER_TYPE=kops IMAGE_NAME=s3-csi-driver TAG=v0.1.0 tests/e2e-kubernetes/run.sh
+
+ACTION=uninstall_driver AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh
+
+ACTION=delete_cluster AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh
+```
+
+## Running tests
+### On cluster created by run.sh
+`run_tests` command is expected to be executed from tests/e2e-kubernetes directory:
+```bash
+pushd tests/e2e-kubernetes
+ACTION=run_tests AWS_REGION=us-east-1 CLUSTER_TYPE=kops TAG=v0.1.0 ./run.sh
+popd
+```
+
+### On existing cluster
 From repository root:
 ```
 make e2e E2E_KUBECONFIG=~/.kube/config E2E_REGION=eu-west-1
 ```
+> E2E_REGION specifies where to create bucket for test (should be the same as where cluster is located)
 
-## Prerequisites
+#### Prerequisites
 - existing k8s cluster (e.g. EKS)
 - `kubectl` in $PATH
 - `kubeconfig` setting up access to k8s cluster
 - driver deployed in the cluster
-- aws credentials with access to s3 (create/delete buckets, read/write/list objects)
+- aws credentials with access to s3 (create/delete buckets, read/write/list objects)
diff --git a/tests/e2e-kubernetes/eksctl.sh b/tests/e2e-kubernetes/eksctl.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euox pipefail
+
+# does not actually create cluster yet
+function eksctl_create_cluster() {
+    EKS_CLUSTER_NAME=${1}
+    EKS_REGION=${2}
+    KUBECONFIG=${3}
+    aws eks update-kubeconfig --region ${EKS_REGION} --name ${EKS_CLUSTER_NAME} --kubeconfig=${KUBECONFIG}
+}
+
+function eksctl_delete_cluster() {
+    echo "eksctl is still using pre-created cluster"
+}
diff --git a/tests/e2e-kubernetes/helm.sh b/tests/e2e-kubernetes/helm.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -euox pipefail
+
+function helm_install() {
+  INSTALL_PATH=${1}
+  if [[ ! -e ${INSTALL_PATH}/helm ]]; then
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+    chmod 700 get_helm.sh
+    export USE_SUDO=false
+    export HELM_INSTALL_DIR=${INSTALL_PATH}
+    ./get_helm.sh
+    rm get_helm.sh
+  fi
+}
+
+function helm_uninstall_driver() {
+  HELM_BIN=${1}
+  KUBECTL_BIN=${2}
+  RELEASE_NAME=${3}
+  KUBECONFIG=${4}
+  if [[ $($HELM_BIN list -A --kubeconfig $KUBECONFIG | grep $RELEASE_NAME) == *deployed* ]]; then
+    $HELM_BIN uninstall $RELEASE_NAME --namespace kube-system --kubeconfig $KUBECONFIG
+    $KUBECTL_BIN wait --for=delete pod --selector="app=s3-csi-node" -n kube-system --timeout=60s --kubeconfig $KUBECONFIG
+  else
+    echo "driver does not seem to be installed"
+  fi
+  $KUBECTL_BIN get pods -A --kubeconfig $KUBECONFIG
+  $KUBECTL_BIN get CSIDriver --kubeconfig $KUBECONFIG
+}
+
+function helm_install_driver() {
+  HELM_BIN=${1}
+  KUBECTL_BIN=${2}
+  RELEASE_NAME=${3}
+  REPOSITORY=${4}
+  TAG=${5}
+  KUBECONFIG=${6}
+  helm_uninstall_driver \
+    "$HELM_BIN" \
+    "$KUBECTL_BIN" \
+    "$RELEASE_NAME" \
+    "$KUBECONFIG"
+  # temporary crutch to make eksctl working with pre-created cluster
+  SA_CREATE=true
+  if [[ "${KUBECONFIG}" == *"s3-csi-cluster.kubeconfig"* ]]; then
+    SA_CREATE=false
+  fi
+  $HELM_BIN upgrade --install $RELEASE_NAME --namespace kube-system ./charts/aws-s3-csi-driver --values \
+    ./charts/aws-s3-csi-driver/values.yaml \
+    --set image.repository=${REPOSITORY} \
+    --set image.tag=${TAG} \
+    --set image.pullPolicy=Always \
+    --set node.serviceAccount.create=${SA_CREATE} \
+    --kubeconfig ${KUBECONFIG}
+  $KUBECTL_BIN rollout status daemonset s3-csi-node -n kube-system --timeout=60s --kubeconfig $KUBECONFIG
+  $KUBECTL_BIN get pods -A --kubeconfig $KUBECONFIG
+  echo "s3-csi-node-image: $($KUBECTL_BIN get daemonset s3-csi-node -n kube-system -o jsonpath="{$.spec.template.spec.containers[:1].image}" --kubeconfig $KUBECONFIG)"
+}
diff --git a/tests/e2e-kubernetes/install.sh b/tests/e2e-kubernetes/install.sh
diff --git a/tests/e2e-kubernetes/kops-patch-node.yaml b/tests/e2e-kubernetes/kops-patch-node.yaml
@@ -0,0 +1,3 @@
+spec:
+  instanceMetadata:
+    httpPutResponseHopLimit: 3
diff --git a/tests/e2e-kubernetes/kops-patch.yaml b/tests/e2e-kubernetes/kops-patch.yaml
@@ -0,0 +1,12 @@
+spec:
+  additionalPolicies:
+    node: |
+      [
+        {
+          "Effect": "Allow",
+          "Action": [
+            "s3:*"
+          ],
+          "Resource": "*"
+        }
+      ]