Add testing to GA

StephenSorriaux · Mar 11, 2023 · d5d1aec · d5d1aec
1 parent 554019b
commit d5d1aec
Show file tree

Hide file tree

Showing 3 changed files with 93 additions and 28 deletions.
diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
@@ -21,32 +21,50 @@ jobs:
         include:
         - scala: 2.12
           kafka_version: 2.1.1
-        - scala: 2.12
-          kafka_version: 2.2.2
-        - scala: 2.12
-          kafka_version: 2.3.1
-        - scala: 2.12
-          kafka_version: 2.4.1
-        - scala: 2.12
-          kafka_version: 2.5.1
-        - scala: 2.13
-          kafka_version: 2.6.3
-        - scala: 2.13
-          kafka_version: 2.7.2
-        - scala: 2.13
-          kafka_version: 2.8.2
-        - scala: 2.13
-          kafka_version: 3.0.2
-        - scala: 2.13
-          kafka_version: 3.1.2
-        - scala: 2.13
-          kafka_version: 3.2.3
-        - scala: 2.13
-          kafka_version: 3.3.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.12
+        #   kafka_version: 2.2.2
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.12
+        #   kafka_version: 2.3.1
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.12
+        #   kafka_version: 2.4.1
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.12
+        #   kafka_version: 2.5.1
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.13
+        #   kafka_version: 2.6.3
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.13
+        #   kafka_version: 2.7.2
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.13
+        #   kafka_version: 2.8.2
+        #   extra_compose_file: "test/docker-compose-kafka.yml"
+        # - scala: 2.13
+        #   kafka_version: 3.0.2
+        #   extra_compose_file: "test/docker-compose-kafka3.yml"
+        # - scala: 2.13
+        #   kafka_version: 3.1.2
+        #   extra_compose_file: "test/docker-compose-kafka3.yml"
+        # - scala: 2.13
+        #   kafka_version: 3.2.3
+        #   extra_compose_file: "test/docker-compose-kafka3.yml"
+        # - scala: 2.13
+        #   kafka_version: 3.3.2
+        #   extra_compose_file: "test/docker-compose-kafka3.yml"
         - scala: 2.13
           kafka_version: 3.4.0
+          extra_compose_file: "test/docker-compose-kafka3.yml"
 
     steps:
+    - name: Install needed libs
+      run: |
+        sudo apt update
+        sudo apt install -y shellcheck
+
     - name: Checkout
       uses: actions/checkout@v3
 
@@ -65,9 +83,56 @@ jobs:
       id: setup
       run: |
         echo BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGE_VERSION=${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_ENV
         echo DOCKER_IMAGE_TAG=${{ env.IMAGE_DEST }}:${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_OUTPUT
 
-    - name: Build and push
+    - name: Build image
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        platforms: linux/amd64,linux/arm64
+        push: false
+        tags: ${{ steps.setup.outputs.DOCKER_IMAGE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          kafka_version=${{ matrix.kafka_version }}
+          scala_version=${{ matrix.scala }}
+          vcs_ref=${{ github.sha }}
+          build_date=${{ steps.setup.outputs.BUILD_DATE }}
+
+    - name: Create test env using docker-compose
+      run: |
+        docker-compose -f test/docker-compose.yml -f ${{ matrix.extra_compose_file }} up -d zookeeper kafka_1 kafka_2
+
+    - name: Perform shellcheck
+      run: |
+        echo "Shellcheck main source files"
+        shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
+        cd test
+        echo "Shellcheck the tests"
+        shellcheck -x -e SC1090 -s bash *.sh **/*.sh
+
+    - name: Perform tests for image
+      run: |
+        cd test
+        echo "Verify docker image's label"
+        ./verifyImageLabels.sh
+        echo "Wait for containers to start"
+        sleep 5
+        # docker-compose logs
+        docker ps -a
+        echo "Run tests"
+        ./runAllTests.sh
+        echo "End-to-End scenario tests"
+        cd scenarios
+        ./runJmxScenario.sh
+
+    - name: Destroy test env using docker-compose
+      run: |
+        docker-compose down
+
+    - name: Push image
       uses: docker/build-push-action@v4
       with:
         context: .
@@ -80,4 +145,4 @@ jobs:
           kafka_version=${{ matrix.kafka_version }}
           scala_version=${{ matrix.scala }}
           vcs_ref=${{ github.sha }}
-          build_date=${{ steps.setup.outputs.BUILD_DATE }}
+          build_date=${{ steps.setup.outputs.BUILD_DATE }}
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.1'
 
 x-kafka-defaults: &kafka-defaults
-  image: wurstmeister/kafka
+  image: "ssorriaux/kafka:${DOCKER_IMAGE_VERSION}"
   ports:
     - "9092"
   volumes:
@@ -24,7 +24,7 @@ services:
       KAFKA_BROKER_ID: 2
 
   kafkatest:
-    image: wurstmeister/kafka
+    image: "ssorriaux/kafka:${DOCKER_IMAGE_VERSION}"
     environment:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:

diff --git a/test/verifyImageLabels.sh b/test/verifyImageLabels.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' wurstmeister/kafka)
+VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' ssorriaux/kafka:"${DOCKER_IMAGE_VERSION}")
 echo "VCS_REF=$VCS_REF"
 if [ -z "$VCS_REF" ] || [ "$VCS_REF" = "unspecified" ]; then
   echo "org.label-schema.vcs-ref is empty or unspecified"
@@ -11,7 +11,7 @@ if ! git cat-file -e "$VCS_REF^{commit}"; then
   exit 1
 fi
 
-BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' wurstmeister/kafka)
+BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' ssorriaux/kafka:"${DOCKER_IMAGE_VERSION}")
 echo "BUILD_DATE=$BUILD_DATE"
 if ! date -d "$BUILD_DATE"; then
   echo "$BUILD_DATE Not a valid date"