Add support for kafka 3 with zookeeper support (adding images for 3.0.1, 3.1.1 and 3.2.0)

.travis.yml

@@ -8,27 +8,56 @@ services:
 # This version will be also tagged as 'latest'
 env:
   global:
-    - LATEST="2.13-2.8.1"
+    - LATEST="2.13-3.2.0"
 
 # Build recommended versions based on: http://kafka.apache.org/downloads
 matrix:
   include:
   - scala: 2.12
-    env: KAFKA_VERSION=2.1.1
+    env:
+      - KAFKA_VERSION=2.1.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.2.2
+    env:
+      - KAFKA_VERSION=2.2.2
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.3.1
+    env:
+      - KAFKA_VERSION=2.3.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.4.1
+    env:
+      - KAFKA_VERSION=2.4.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.5.1
+    env:
+      - KAFKA_VERSION=2.5.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.6.3
+    env:
+      - KAFKA_VERSION=2.6.3
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.7.2
+    env:
+      - KAFKA_VERSION=2.7.2
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.8.1
+    env:
+      - KAFKA_VERSION=2.8.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
+  - scala: 2.13
+    env:
+      - KAFKA_VERSION=3.0.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+  - scala: 2.13
+    env:
+      - KAFKA_VERSION=3.1.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+  - scala: 2.13
+    env:
+      - KAFKA_VERSION=3.2.0
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+
 
 # Upgrade Docker Engine so we can use buildx
 before_install:
@@ -60,7 +89,7 @@ install:
   - docker pull confluentinc/cp-kafkacat
 
 before_script:
-  - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2
+  - docker-compose -f test/docker-compose.yml -f ${EXTRA_COMPOSE_FILE} up -d zookeeper kafka_1 kafka_2
 
 script:
   # Shellcheck main source files

CHANGELOG.md

@@ -2,7 +2,10 @@ Changelog
 =========
 
 Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date.
+20-May-2022
+-----------
 
+- Add support for Kafka `3.0.1`,`3.1.1` and `3.2.0`
 09-Apr-2022
 ----------
 

Dockerfile

@@ -1,6 +1,6 @@
-FROM openjdk:11-jre-slim
+FROM eclipse-temurin:17-jre
 
-ARG kafka_version=2.8.1
+ARG kafka_version=3.4.0
 ARG scala_version=2.13
 ARG vcs_ref=unspecified
 ARG build_date=unspecified
@@ -28,9 +28,9 @@ RUN set -eux ; \
     apt-get install -y --no-install-recommends jq net-tools curl wget ; \
 ### BEGIN docker for CI tests
     apt-get install -y --no-install-recommends gnupg lsb-release ; \
-	curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg ; \
+	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg ; \
 	echo \
-  		"deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
+  		"deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
   		$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null ; \
     apt-get update ; \
     apt-get install -y --no-install-recommends docker-ce-cli ; \

README.md

@@ -89,7 +89,7 @@ For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to sp
 
 ## Advertised hostname
 
-You can configure the advertised hostname in different ways
+You can configure the advertised hostname in different ways (Note that as of kafka 3 KAFKA_ADVERTISED_HOST_NAME,KAFKA_ADVERTISED_PORT KAFKA_PORT and KAFKA_HOST_NAME is no longer supported)
 
 1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME```
 2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"```

broker-list.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}')
-BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
+# We are just interested in ipv4 addresses, so ignore ipv6 addresses
+BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | grep '0.0.0.0' | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
 echo "${BROKERS//$'\n'/,}"

create-topics.sh

@@ -11,24 +11,41 @@ fi
 start_timeout_exceeded=false
 count=0
 step=10
-while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do
-    echo "waiting for kafka to be ready"
-    sleep $step;
-    count=$((count + step))
-    if [ $count -gt $START_TIMEOUT ]; then
-        start_timeout_exceeded=true
-        break
+
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option.
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    if [[ -v KAFKA_LISTENERS ]]; then
+        PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' )
+        CONNECT_OPTS="--bootstrap-server localhost:${PORT}"
+    else
+        CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}"
     fi
-done
+else
+    PORT="$KAFKA_PORT"
+    CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+fi
+
+if [[ ! -v PORT ]]; then
+    while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do
+        echo "waiting for kafka to be ready"
+        sleep $step;
+        count=$((count + step))
+        if [ $count -gt $START_TIMEOUT ]; then
+            start_timeout_exceeded=true
+            break
+        fi
+    done
+fi
 
 if $start_timeout_exceeded; then
     echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
     exit 1
 fi
 
 # introduced in 0.10. In earlier versions, this will fail because the topic already exists.
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
 if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then
     KAFKA_0_10_OPTS="--if-not-exists"
 fi
@@ -45,7 +62,7 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP
 
     COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\
 		--create \\
-		--zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
+		${CONNECT_OPTS} \\
 		--topic ${topicConfig[0]} \\
 		--partitions ${topicConfig[1]} \\
 		--replication-factor ${topicConfig[2]} \\

docker-compose-single-broker.yml

@@ -9,7 +9,8 @@ services:
     ports:
       - "9092:9092"
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
       KAFKA_CREATE_TOPICS: "test:1:1"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:

docker-compose.yml

@@ -12,8 +12,8 @@ services:
       - "9092"
     environment:
       DOCKER_API_VERSION: 1.22
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     restart: unless-stopped

start-kafka.sh

@@ -1,5 +1,8 @@
 #!/bin/bash -e
 
+# shellcheck disable=SC1091
+source /usr/bin/versions.sh
+
 # Allow specific kafka versions to perform any unique bootstrap operations
 OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh"
 if [[ -x "$OVERRIDE_FILE" ]]; then
@@ -10,24 +13,45 @@ fi
 # Store original IFS config, so we can restore it at various stages
 ORIG_IFS=$IFS
 
-if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
-    echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"
-    exit 1
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    if [[ -v KAFKA_ADVERTISED_HOST_NAME ]]; then
+        echo "ERROR: KAFKA_ADVERTISED_HOST_NAME is removed as of kafka 3, remove KAFKA_ADVERTISED_HOST_NAME=$KAFKA_ADVERTISED_HOST_NAME from your config"
+        exit 1
+    fi
+    if [[ -v KAFKA_ADVERTISED_PORT ]]; then
+        echo "ERROR: KAFKA_ADVERTISED_PORT is removed as of kafka 3, remove KAFKA_ADVERTISED_PORT=$KAFKA_ADVERTISED_PORT from your config"
+        exit 1
+    fi
+    if [[ -v KAFKA_PORT ]]; then
+        echo "ERROR: KAFKA_PORT is removed as of kafka 3, remove KAFKA_PORT=$KAFKA_PORT from your config"
+        exit 1
+    fi
+    if [[ -v KAFKA_HOST_NAME ]]; then
+        echo "ERROR: KAFKA_HOST_NAME is removed as of kafka 3, remove KAFKA_HOST_NAME=$KAFKA_HOST_NAME from your config"
+        exit 1
+    fi
+else
+    if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
+        echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"
+        exit 1
+    fi
+    if [[ -z "$KAFKA_PORT" ]]; then
+        export KAFKA_PORT=9092
+    fi
 fi
 
-if [[ -z "$KAFKA_PORT" ]]; then
-    export KAFKA_PORT=9092
-fi
 
 create-topics.sh &
 unset KAFKA_CREATE_TOPICS
 
-if [[ -z "$KAFKA_ADVERTISED_PORT" && \
-  -z "$KAFKA_LISTENERS" && \
-  -z "$KAFKA_ADVERTISED_LISTENERS" && \
-  -S /var/run/docker.sock ]]; then
-    KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) 
-    export KAFKA_ADVERTISED_PORT
+if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+    if [[ -z "$KAFKA_ADVERTISED_PORT" && \
+      -z "$KAFKA_LISTENERS" && \
+      -z "$KAFKA_ADVERTISED_LISTENERS" && \
+      -S /var/run/docker.sock ]]; then
+        KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1)
+        export KAFKA_ADVERTISED_PORT
+    fi
 fi
 
 if [[ -z "$KAFKA_BROKER_ID" ]]; then
@@ -81,19 +105,30 @@ if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then
 fi
 
 # Try and configure minimal settings or exit with error if there isn't enough information
-if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then
-    if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then
-        echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS"
-        exit 1
-    elif [[ -z "$HOSTNAME_VALUE" ]]; then
-        echo "ERROR: No listener or advertised hostname configuration provided in environment."
-        echo "       Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME"
-        exit 1
-    fi
+if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+    if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then
+        if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+            echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS"
+            exit 1
+        elif [[ -z "$HOSTNAME_VALUE" ]]; then
+            echo "ERROR: No listener or advertised hostname configuration provided in environment."
+            echo "       Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME"
+            exit 1
+        fi
 
-    # Maintain existing behaviour
-    # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined.
-    export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE"
+        # Maintain existing behaviour
+        # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined.
+        export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE"
+    fi
+else
+    if [[ "$HOSTNAME_VALUE" ]]; then
+        if [[ -z "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+            export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$HOSTNAME_VALUE:9092"
+            if [[ -z "$KAFKA_LISTENERS" ]]; then
+                export KAFKA_LISTENERS=PLAINTEXT://:9092
+            fi
+        fi
+    fi
 fi
 
 #Issue newline to config file in case there is not one already

test/0.0/test.create-topics-custom-separator.kafka.sh

@@ -20,13 +20,25 @@ testCreateTopicsCustomSeparator() {
 
 	create-topics.sh
 
+	# shellcheck disable=SC1091
+	source "/usr/bin/versions.sh"
+
+	# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+	# --bootstrap-server option with a random broker
+	if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+		CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)"
+	else
+		CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+	fi
+
 	# Loop through each array, validate that topic exists
 	for i in "${!TOPICS[@]}"; do
 		TOPIC=${TOPICS[i]}
 
 		echo "Validating topic '$TOPIC'"
 
-		EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC")
+		# shellcheck disable=SC2086
+		EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC")
 		if [[ "$EXISTS" != "$TOPIC" ]]; then
 			echo "$TOPIC topic not created"
 			return 1
@@ -36,8 +48,18 @@ testCreateTopicsCustomSeparator() {
 	return 0
 }
 
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+
+# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}')
+else
+    PORT="$KAFKA_PORT"
+fi
+
 # mock the netstat call as made by the create-topics.sh script
-function netstat() { echo "1 2 3 :$KAFKA_PORT"; }
+function netstat() { echo "1 2 3 :$PORT"; }
 export -f netstat
 
 testCreateTopicsCustomSeparator

test/0.0/test.start-kafka-advertised-host.kafka.sh

@@ -16,5 +16,14 @@ testAdvertisedHost() {
 	assertAbsent 'advertised.listeners'
 	assertAbsent 'listeners'
 }
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
 
-testAdvertisedHost
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    echo "with kafka from version 3.0.0 'advertised.port' and 'advertised.host.name' are removed, making this test obosolete"
+    echo "See: https://github.com/apache/kafka/pull/10872"
+else
+    testAdvertisedHost
+fi