Add 3.X images

.github/workflows/package.yml

@@ -8,7 +8,7 @@ on:
     branches: ["master"]
 
 env:
-  LATEST: "2.13-2.8.1"
+  LATEST: "2.13-3.2.0"
   IMAGE_DEST: "ssorriaux/kafka"
 
 jobs:
@@ -21,22 +21,50 @@ jobs:
         include:
         - scala: 2.12
           kafka_version: 2.1.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.2.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.3.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.4.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.5.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
           kafka_version: 2.6.3
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
           kafka_version: 2.7.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
-          kafka_version: 2.8.1
+          kafka_version: 2.8.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
+        - scala: 2.13
+          kafka_version: 3.0.2
+          extra_compose_file: "test/docker-compose-kafka3.yml"
+        - scala: 2.13
+          kafka_version: 3.1.2
+          extra_compose_file: "test/docker-compose-kafka3.yml"
+        - scala: 2.13
+          kafka_version: 3.2.3
+          extra_compose_file: "test/docker-compose-kafka3.yml"
+        - scala: 2.13
+          kafka_version: 3.3.2
+          extra_compose_file: "test/docker-compose-kafka3.yml"
+        - scala: 2.13
+          kafka_version: 3.4.0
+          extra_compose_file: "test/docker-compose-kafka3.yml"
 
     steps:
+    - name: Install needed libs
+      run: |
+        sudo apt update
+        sudo apt install -y shellcheck
+
     - name: Checkout
       uses: actions/checkout@v3
 
@@ -55,9 +83,56 @@ jobs:
       id: setup
       run: |
         echo BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGE_VERSION=${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_ENV
         echo DOCKER_IMAGE_TAG=${{ env.IMAGE_DEST }}:${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_OUTPUT
 
-    - name: Build and push
+    - name: Build image
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        platforms: linux/amd64,linux/arm64
+        push: false
+        tags: ${{ steps.setup.outputs.DOCKER_IMAGE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          kafka_version=${{ matrix.kafka_version }}
+          scala_version=${{ matrix.scala }}
+          vcs_ref=${{ github.sha }}
+          build_date=${{ steps.setup.outputs.BUILD_DATE }}
+
+    - name: Create test env using docker-compose
+      run: |
+        docker-compose -f test/docker-compose.yml -f ${{ matrix.extra_compose_file }} up -d zookeeper kafka_1 kafka_2
+
+    - name: Perform shellcheck
+      run: |
+        echo "Shellcheck main source files"
+        shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
+        cd test
+        echo "Shellcheck the tests"
+        shellcheck -x -e SC1090 -s bash *.sh **/*.sh
+
+    - name: Perform tests for image
+      run: |
+        cd test
+        echo "Verify docker image's label"
+        ./verifyImageLabels.sh
+        echo "Wait for containers to start"
+        sleep 5
+        # docker-compose logs
+        docker ps -a
+        echo "Run tests"
+        ./runAllTests.sh
+        echo "End-to-End scenario tests"
+        cd scenarios
+        ./runJmxScenario.sh
+
+    - name: Destroy test env using docker-compose
+      run: |
+        docker-compose down
+
+    - name: Push image
       uses: docker/build-push-action@v4
       with:
         context: .
@@ -70,4 +145,4 @@ jobs:
           kafka_version=${{ matrix.kafka_version }}
           scala_version=${{ matrix.scala }}
           vcs_ref=${{ github.sha }}
-          build_date=${{ steps.setup.outputs.BUILD_DATE }}
+          build_date=${{ steps.setup.outputs.BUILD_DATE }}

.travis.yml

@@ -8,27 +8,56 @@ services:
 # This version will be also tagged as 'latest'
 env:
   global:
-    - LATEST="2.13-2.8.1"
+    - LATEST="2.13-3.2.0"
 
 # Build recommended versions based on: http://kafka.apache.org/downloads
 matrix:
   include:
   - scala: 2.12
-    env: KAFKA_VERSION=2.1.1
+    env:
+      - KAFKA_VERSION=2.1.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.2.2
+    env:
+      - KAFKA_VERSION=2.2.2
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.3.1
+    env:
+      - KAFKA_VERSION=2.3.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.4.1
+    env:
+      - KAFKA_VERSION=2.4.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.5.1
+    env:
+      - KAFKA_VERSION=2.5.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.6.3
+    env:
+      - KAFKA_VERSION=2.6.3
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.7.2
+    env:
+      - KAFKA_VERSION=2.7.2
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.8.1
+    env:
+      - KAFKA_VERSION=2.8.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
+  - scala: 2.13
+    env:
+      - KAFKA_VERSION=3.0.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+  - scala: 2.13
+    env:
+      - KAFKA_VERSION=3.1.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+  - scala: 2.13
+    env:
+      - KAFKA_VERSION=3.2.0
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+
 
 # Upgrade Docker Engine so we can use buildx
 before_install:
@@ -60,7 +89,7 @@ install:
   - docker pull confluentinc/cp-kafkacat
 
 before_script:
-  - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2
+  - docker-compose -f test/docker-compose.yml -f ${EXTRA_COMPOSE_FILE} up -d zookeeper kafka_1 kafka_2
 
 script:
   # Shellcheck main source files

CHANGELOG.md

@@ -2,7 +2,10 @@ Changelog
 =========
 
 Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date.
+20-May-2022
+-----------
 
+- Add support for Kafka `3.0.1`,`3.1.1` and `3.2.0`
 09-Apr-2022
 ----------
 

Dockerfile

@@ -1,6 +1,6 @@
 FROM openjdk:11-jre-slim
 
-ARG kafka_version=2.8.1
+ARG kafka_version=3.2.0
 ARG scala_version=2.13
 ARG vcs_ref=unspecified
 ARG build_date=unspecified

README.md

@@ -1,24 +1,26 @@
-[![Docker Pulls](https://img.shields.io/docker/pulls/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/)
-[![Docker Stars](https://img.shields.io/docker/stars/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/)
-[![](https://images.microbadger.com/badges/version/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own version badge on microbadger.com")
-[![](https://images.microbadger.com/badges/image/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own image badge on microbadger.com")
-[![Build Status](https://app.travis-ci.com/wurstmeister/kafka-docker.svg?branch=master)](https://app.travis-ci.com/wurstmeister/kafka-docker)
+[![Docker Pulls](https://img.shields.io/docker/pulls/ssorriaux/kafka.svg)](https://hub.docker.com/r/ssorriaux/kafka/)
+[![Docker Stars](https://img.shields.io/docker/stars/ssorriaux/kafka.svg)](https://hub.docker.com/r/ssorriaux/kafka/)
+[![](https://images.microbadger.com/badges/version/ssorriaux/kafka.svg)](https://microbadger.com/images/ssorriaux/kafka "Get your own version badge on microbadger.com")
+[![](https://images.microbadger.com/badges/image/ssorriaux/kafka.svg)](https://microbadger.com/images/ssorriaux/kafka "Get your own image badge on microbadger.com")
+[![Build Status](https://app.travis-ci.com/ssorriaux/kafka-docker.svg?branch=master)](https://app.travis-ci.com/ssorriaux/kafka-docker)
 
 
 kafka-docker
 ============
 
+**Fork of [wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) since no update was provided since May, 2022.**
+
 Dockerfile for [Apache Kafka](http://kafka.apache.org/)
 
-The image is available directly from [Docker Hub](https://hub.docker.com/r/wurstmeister/kafka/)
+The image is available directly from [Docker Hub](https://hub.docker.com/r/ssorriaux/kafka/)
 
 Tags and releases
 -----------------
 
 All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `<scala version>-<kafka version>`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). To list all available tags:
 
 ```
-curl -s https://registry.hub.docker.com/v2/repositories/wurstmeister/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*'
+curl -s https://registry.hub.docker.com/v2/repositories/ssorriaux/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*'
 ```
 
 Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image.
@@ -34,7 +36,7 @@ Everytime the image is updated, all tags will be pushed with the latest updates.
 ## Pre-Requisites
 
 - install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/)
-- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/wurstmeister/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.)
+- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/ssorriaux/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.)
 - if you want to customize any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'```
 - Kafka's log4j usage can be customized by adding environment variables prefixed with ```LOG4J_```. These will be mapped to ```log4j.properties```. For example: ```LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER=DEBUG, authorizerAppender```
 
@@ -56,7 +58,7 @@ Destroy a cluster:
 
 ## Note
 
-The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/wurstmeister/kafka-docker/blob/master/docker-compose-single-broker.yml):
+The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/StephenSorriaux/kafka-docker/blob/master/docker-compose-single-broker.yml):
 
 - ```docker-compose -f docker-compose-single-broker.yml up```
 
@@ -89,7 +91,7 @@ For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to sp
 
 ## Advertised hostname
 
-You can configure the advertised hostname in different ways
+You can configure the advertised hostname in different ways (Note that as of kafka 3 KAFKA_ADVERTISED_HOST_NAME,KAFKA_ADVERTISED_PORT KAFKA_PORT and KAFKA_HOST_NAME is no longer supported)
 
 1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME```
 2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"```

broker-list.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}')
-BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
+# We are just interested in ipv4 addresses, so ignore ipv6 addresses
+BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | grep '0.0.0.0' | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
 echo "${BROKERS//$'\n'/,}"

create-topics.sh

@@ -11,24 +11,41 @@ fi
 start_timeout_exceeded=false
 count=0
 step=10
-while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do
-    echo "waiting for kafka to be ready"
-    sleep $step;
-    count=$((count + step))
-    if [ $count -gt $START_TIMEOUT ]; then
-        start_timeout_exceeded=true
-        break
+
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option.
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    if [[ -v KAFKA_LISTENERS ]]; then
+        PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' )
+        CONNECT_OPTS="--bootstrap-server localhost:${PORT}"
+    else
+        CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}"
     fi
-done
+else
+    PORT="$KAFKA_PORT"
+    CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+fi
+
+if [[ ! -v PORT ]]; then
+    while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do
+        echo "waiting for kafka to be ready"
+        sleep $step;
+        count=$((count + step))
+        if [ $count -gt $START_TIMEOUT ]; then
+            start_timeout_exceeded=true
+            break
+        fi
+    done
+fi
 
 if $start_timeout_exceeded; then
     echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
     exit 1
 fi
 
 # introduced in 0.10. In earlier versions, this will fail because the topic already exists.
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
 if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then
     KAFKA_0_10_OPTS="--if-not-exists"
 fi
@@ -45,7 +62,7 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP
 
     COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\
 		--create \\
-		--zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
+		${CONNECT_OPTS} \\
 		--topic ${topicConfig[0]} \\
 		--partitions ${topicConfig[1]} \\
 		--replication-factor ${topicConfig[2]} \\

docker-compose-single-broker.yml

@@ -9,7 +9,8 @@ services:
     ports:
       - "9092:9092"
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
       KAFKA_CREATE_TOPICS: "test:1:1"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:

docker-compose.yml

@@ -12,8 +12,8 @@ services:
       - "9092"
     environment:
       DOCKER_API_VERSION: 1.22
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     restart: unless-stopped