k8s test matrix via kuttl + upgrade cass-operator to 1.8.0 in tests […

…K8SSAND-1008] [K8SSAND-1013] (#80)

* Add kuttl tests.
* Use kustomize for all kuttl tests.
* Migrate cass-operator in kuttl tests to v 1.8.0.
* Makefile steps for kuttl tests.
* Github actions workflows for kuttl tests.
* Reduce kind worker count down to 1 for all tests.
* Add additional kind configs to allow testing on more k8s versions.
* No multiple nodes per worker. Test only requires one node.
* Kind logs in .gitignore.
* Add 1.20-1.22 k8s versions into a test matrix for kuttl tests.

.github/workflows/build.yml

@@ -9,7 +9,7 @@ on:
     branches: [ master ]
 
 jobs:
-  testing:
+  unit_tests:
     name: Run unit tests
     runs-on: ubuntu-latest
     env:
@@ -35,7 +35,7 @@ jobs:
           make test
   build_docker_image:
     runs-on: ubuntu-latest
-    needs: testing
+    needs: unit_tests
     name: Build reaper-operator image
     steps:
       - name: Check out code into the Go module directory
@@ -70,7 +70,7 @@ jobs:
           path: /tmp/k8ssandra-reaper-operator.tar
   run_e2e_test:
     runs-on: ubuntu-latest
-    needs: [testing, build_docker_image]
+    needs: [unit_tests, build_docker_image]
     name: Run e2e tests
     steps:
       - name: Set up Go 1.x
@@ -92,9 +92,9 @@ jobs:
         uses: helm/[email protected]
         with:
           version: v0.11.1
-          node_image: kindest/node:v1.20.2
+          node_image: kindest/node:v1.21.2
           cluster_name: kind
-          config: test/config/kind/kind_config_3_workers.yaml
+          config: test/config/kind/w1k1.21.yaml
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Download reaper-operator image
@@ -112,9 +112,39 @@ jobs:
         run: |
           export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
           make e2e-test
+  run_kuttl_test:
+    runs-on: ubuntu-latest
+    needs: [unit_tests, build_docker_image]
+    name: Run kuttl tests
+    strategy:
+      matrix:
+        k8s_version: ["1.22", "1.21", "1.20"]
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.15
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Kind kube-proxy issue workaround
+      run: sudo sysctl net/netfilter/nf_conntrack_max=524288
+    - name: Download reaper-operator image
+      uses: actions/download-artifact@v2
+      with:
+        name: reaper-operator
+        path: /tmp
+    - name: Load Docker images
+      run: |
+        docker load --input /tmp/k8ssandra-reaper-operator.tar
+    - name: Run kuttl-test
+      run: |
+        # We are running tests against k8s 1.20 - 1.22 currently. 
+        # Additional versions must be added in kind config files under ./test/config/kind
+        make KUTTL_KIND_CFG="./test/config/kind/w1k${{ matrix.k8s_version }}.yaml" kuttl-test
   push_image:
     runs-on: ubuntu-latest
-    needs: run_e2e_test
+    needs: [run_e2e_test, run_kuttl_test]
     if: github.ref == 'refs/heads/master'
     name: Push image
     steps:

.gitignore

@@ -86,4 +86,6 @@ tags
 
 # forks for running tests with custom images
 test/config/deploy_reaper_test/overlays/forks/
-build/
+
+kind-logs*/
+build/

Makefile

@@ -1,5 +1,5 @@
+.SHELLFLAGS = -e -c
 SHELL := /bin/bash
-
 # Current Operator version
 VERSION ?= 0.0.1
 # Default bundle image tag
@@ -37,10 +37,13 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+KREW_INSTALLED=$(shell kubectl krew; $?)
+KUTTL_KIND_CFG ?= "./test/config/kind/w1k1.21.yaml"
+
 all: manager
 
 # Run tests
-ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+ENVTEST_ASSETS_DIR=$(shell pwd)/testbin/bin
 test: generate fmt vet manifests
 	mkdir -p ${ENVTEST_ASSETS_DIR}
 	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
@@ -55,6 +58,19 @@ manager: generate fmt vet
 run: generate fmt vet manifests
 	go run ./main.go
 
+# E2E tests from kuttl
+kuttl-test: install-kuttl
+	PATH="./testbin/bin:$$PATH"; kubectl-kuttl test --kind-config=${KUTTL_KIND_CFG}
+
+ # Install kuttl for e2e tests.
+install-kuttl: 
+	mkdir -p ./testbin/bin ; \
+	cd ./testbin/bin ; \
+	OS="$$(uname | tr '[:upper:]' '[:lower:]')" ; \
+  	ARCH="$$(uname -m | sed -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$$/arm64/')" ; \
+	curl -LO https://github.com/kudobuilder/kuttl/releases/download/v0.11.1/kuttl_0.11.1_$${OS}_$${ARCH}.tar.gz ; \
+	tar -zxvf kuttl_0.11.1_$${OS}_$${ARCH}.tar.gz ; 
+
 # Install CRDs into a cluster
 install: manifests kustomize
 	$(KUSTOMIZE) build config/crd | kubectl apply -f -
@@ -96,7 +112,7 @@ generate: controller-gen
 # Build the docker image
 docker-build:
 	@echo Building ${REV_IMAGE}
-	docker build -t ${IMG} .
+	docker build -t ${REV_IMAGE} .
 	docker tag ${REV_IMAGE} ${LATEST_IMAGE}
 
 # Push the docker image

controllers/reaper_controller_test.go

@@ -23,8 +23,7 @@ import (
 const (
 	ReaperName              = "test-reaper"
 	CassandraDatacenterName = "test-dc"
-
-	timeout  = time.Second * 20
+	timeout  = time.Second * 100
 	interval = time.Millisecond * 250
 )
 

kuttl-test.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+- ./test/kuttl
+
+startKIND: true
+kindNodeCache: true
+# we are using a k8s 1.22 as default. Other versions can be run using the --kind-config flag.
+kindConfig: test/config/kind/w1k1.22.yaml 
+
+timeout: 1400
+namespace: default
+kindContainers:
+- k8ssandra/reaper-operator:latest

test/config/cassdc/cassdc.yaml

@@ -7,13 +7,12 @@ metadata:
 spec:
   clusterName: reaper-test
   serverType: cassandra
-  serverVersion: 3.11.10
-  serverImage: k8ssandra/cass-management-api:3.11.10-v0.1.25
+  serverVersion: 3.11.11
+  serverImage: k8ssandra/cass-management-api:3.11.11-v0.1.31
   dockerImageRunsAsCassandra: true
   managementApiAuth:
     insecure: {}
   size: 1
-  allowMultipleNodesPerWorker: true
   config:
     jvm-options:
       initial_heap_size: "512m"

test/config/deploy_crds/kustomization.yaml

@@ -1,5 +1,6 @@
 namespace: deploy-reaper-test
 
 resources:
+- namespace.yaml
 - ../../../config/crd
-- ../cass-operator-crd
+- ../cass-operator-crd

test/config/deploy_crds/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: deploy-reaper-test

test/config/deploy_reaper_test/kustomization.yaml

@@ -1,2 +1,2 @@
 resources:
-- base
+- base

test/config/kind/kind_config_3_workers.yaml → test/config/kind/w1k1.20.yaml

@@ -4,6 +4,6 @@ networking:
   apiServerPort: 45451
 nodes:
 - role: control-plane
+  image: kindest/node:v1.20.7
 - role: worker
-- role: worker
-- role: worker
+  image: kindest/node:v1.20.7

test/config/kind/w1k1.21.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+  image: kindest/node:v1.21.2
+- role: worker
+  image: kindest/node:v1.21.2

test/config/kind/w1k1.22.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+  image: kindest/node:v1.22.2
+- role: worker
+  image: kindest/node:v1.22.2

test/config/kind/w3k1.21.yaml

@@ -0,0 +1,12 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerPort: 45451
+nodes:
+- role: control-plane
+- role: worker
+  image: kindest/node:v1.21.2
+- role: worker
+  image: kindest/node:v1.21.2
+- role: worker
+  image: kindest/node:v1.21.2

test/e2e/deploy_reaper_test.go

@@ -29,10 +29,7 @@ const (
 var _ = Describe("Deploy Reaper with Cassandra backend", func() {
 	Context("When a Cassandra cluster is deployed", func() {
 		Specify("Reaper is deployed", func() {
-			By("create namespace " + namespace)
-			err := framework.CreateNamespace(namespace)
-			Expect(err).ToNot(HaveOccurred())
-
+			var err error
 			By("deploy CRDs")
 			framework.KustomizeAndApply(namespace, "deploy_crds", false)
 			framework.WaitForCRDs(namespace)

test/kuttl/test-can-install/00-assert.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-cainjector
+  namespace: "cert-manager"
+  labels:
+    app: cainjector
+    app.kubernetes.io/name: cainjector
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager
+  namespace: "cert-manager"
+  labels:
+    app: cert-manager
+    app.kubernetes.io/name: cert-manager
+    app.kubernetes.io/instance: cert-manager
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-webhook
+  namespace: "cert-manager"
+  labels:
+    app: webhook
+    app.kubernetes.io/name: webhook
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing

test/kuttl/test-can-install/00-cert-manager.yaml

@@ -0,0 +1,6 @@
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- command: kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.0/cert-manager.yaml
+  ignoreFailure: false 

test/kuttl/test-can-install/01-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: cass-operator
+    cassandra.datastax.com/cluster: reaper-test
+    cassandra.datastax.com/datacenter: reaper-test
+    cassandra.datastax.com/node-state: Started
+  name: reaper-test-reaper-test-default-sts-0
+  namespace: default
+status:
+  phase: Running

test/kuttl/test-can-install/01-cass-operator.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- command: kubectl apply -k ./config/cass-operator
+  ignoreFailure: false
+- command: kubectl apply -k ../../config/cassdc
+  ignoreFailure: false

test/kuttl/test-can-install/04-assert.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reaper-operator
+  namespace: default
+status:
+  conditions:
+  - status: "True"
+    type: Available
+  - status: "True"
+    type: Progressing
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: reapers.reaper.cassandra-reaper.io

test/kuttl/test-can-install/04-reaper-operator.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- script: |
+    kubectl apply -k ./config/reaper-operator
+  ignoreFailure: false 

test/kuttl/test-can-install/05-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: reaper-operator # Make sure we have a reaper pod managed by the operator.
+  namespace: default
+status:
+  phase: Running

test/kuttl/test-can-install/05-reaper.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands: 
+- script: |
+    kubectl apply -f ./config/reaper.yaml
+  ignoreFailure: false 

test/kuttl/test-can-install/config/cass-operator/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- github.com/k8ssandra/cass-operator/config/deployments/cluster?ref=v1.8.0

test/kuttl/test-can-install/config/reaper-operator/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: default
+resources:
+- ../../../../../config/crd # we are going to apply this in the parent kuttl suite config so the resource exists. This is here to be patched.
+- ../../../../../config/rbac
+- ../../../../../config/manager