From ac64f789024fa1479985eac237f3694f1a64dabe Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 11 May 2022 15:46:35 +0200
Subject: [PATCH 01/34] Jack12816 initial 3.0 approach

---
 .travis.yml                                   |  4 ++-
 CHANGELOG.md                                  |  5 +++
 Dockerfile                                    |  2 +-
 README.md                                     |  1 +
 broker-list.sh                                |  3 +-
 create-topics.sh                              | 10 +++++-
 start-kafka.sh                                | 15 +++++++-
 ...st.create-topics-custom-separator.kafka.sh | 14 +++++++-
 test/0.10/test.create-topics.kafka.sh         | 36 ++++++++++++++-----
 test/docker-compose.yml                       |  2 +-
 test/runAllTests.sh                           |  4 +--
 11 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6b32acb9..3f67746c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ services:
 # This version will be also tagged as 'latest'
 env:
   global:
-    - LATEST="2.13-2.7.1"
+    - LATEST="2.13-3.0.0"
 
 # Build recommended versions based on: http://kafka.apache.org/downloads
 matrix:
@@ -29,6 +29,8 @@ matrix:
     env: KAFKA_VERSION=2.6.0
   - scala: 2.13
     env: KAFKA_VERSION=2.7.1
+  - scala: 2.13
+    env: KAFKA_VERSION=3.0.0
 
 install:
   - docker --version
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ebefdf17..0fad1d59 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,11 @@ Changelog
 
 Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date.
 
+11-Oct-2021
+-----------
+
+- Add support for Kafka `3.0.0`
+
 19-July-2021
 ----------
 
diff --git a/Dockerfile b/Dockerfile
index 668ac714..753ec337 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
 FROM azul/zulu-openjdk-alpine:8u292-8.54.0.21
 
-ARG kafka_version=2.7.1
+ARG kafka_version=3.0.0
 ARG scala_version=2.13
 ARG glibc_version=2.31-r0
 ARG vcs_ref=unspecified
diff --git a/README.md b/README.md
index 629ddbd9..acc78d90 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,7 @@ Tags and releases
 
 All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `<scala version>-<kafka version>`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). Available tags are:
 
+- `2.13-3.0.0`
 - `2.13-2.7.1`
 - `2.13-2.6.0`
 - `2.12-2.5.0`
diff --git a/broker-list.sh b/broker-list.sh
index 5c5ee2d7..4648330d 100755
--- a/broker-list.sh
+++ b/broker-list.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 
 CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}')
-BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
+# We are just interested in ipv4 addresses, so ignore ipv6 addresses
+BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | grep '0.0.0.0' | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
 echo "${BROKERS//$'\n'/,}"
diff --git a/create-topics.sh b/create-topics.sh
index 0bacf7b5..3ccbe194 100755
--- a/create-topics.sh
+++ b/create-topics.sh
@@ -33,6 +33,14 @@ if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSI
     KAFKA_0_10_OPTS="--if-not-exists"
 fi
 
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)"
+else
+    CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+fi
+
 # Expected format:
 #   name:partitions:replicas:cleanup.policy
 IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do
@@ -45,7 +53,7 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP
 
     COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\
 		--create \\
-		--zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
+		${CONNECT_OPTS} \\
 		--topic ${topicConfig[0]} \\
 		--partitions ${topicConfig[1]} \\
 		--replication-factor ${topicConfig[2]} \\
diff --git a/start-kafka.sh b/start-kafka.sh
index b48280a4..fb56deb5 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -1,5 +1,8 @@
 #!/bin/bash -e
 
+# shellcheck disable=SC1091
+source /usr/bin/versions.sh
+
 # Allow specific kafka versions to perform any unique bootstrap operations
 OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh"
 if [[ -x "$OVERRIDE_FILE" ]]; then
@@ -26,7 +29,7 @@ if [[ -z "$KAFKA_ADVERTISED_PORT" && \
   -z "$KAFKA_LISTENERS" && \
   -z "$KAFKA_ADVERTISED_LISTENERS" && \
   -S /var/run/docker.sock ]]; then
-    KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) 
+    KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1)
     export KAFKA_ADVERTISED_PORT
 fi
 
@@ -96,6 +99,16 @@ if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then
     export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE"
 fi
 
+# `advertised.port` and `advertised.host.name` are removed with Kafka 3.0.0
+# See: https://github.com/apache/kafka/pull/10872
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    if [ -z "$KAFKA_ADVERTISED_LISTENERS" ]; then
+        if [ -n "${KAFKA_ADVERTISED_HOST_NAME}" ]; then
+            export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:${KAFKA_PORT}"
+        fi
+    fi
+fi
+
 #Issue newline to config file in case there is not one already
 echo "" >> "$KAFKA_HOME/config/server.properties"
 
diff --git a/test/0.0/test.create-topics-custom-separator.kafka.sh b/test/0.0/test.create-topics-custom-separator.kafka.sh
index abd031c1..4ad32bd3 100755
--- a/test/0.0/test.create-topics-custom-separator.kafka.sh
+++ b/test/0.0/test.create-topics-custom-separator.kafka.sh
@@ -20,13 +20,25 @@ testCreateTopicsCustomSeparator() {
 
 	create-topics.sh
 
+	# shellcheck disable=SC1091
+	source "/usr/bin/versions.sh"
+
+	# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+	# --bootstrap-server option with a random broker
+	if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+		CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)"
+	else
+		CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+	fi
+
 	# Loop through each array, validate that topic exists
 	for i in "${!TOPICS[@]}"; do
 		TOPIC=${TOPICS[i]}
 
 		echo "Validating topic '$TOPIC'"
 
-		EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC")
+		# shellcheck disable=SC2086
+		EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC")
 		if [[ "$EXISTS" != "$TOPIC" ]]; then
 			echo "$TOPIC topic not created"
 			return 1
diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh
index e9dda7bd..d0088291 100755
--- a/test/0.10/test.create-topics.kafka.sh
+++ b/test/0.10/test.create-topics.kafka.sh
@@ -8,27 +8,47 @@ testCreateTopics() {
 	# TOPICS array contains the topic name to create / validate
 	# CLEANUP array contains the expected cleanup policy configuration for the topic
 	TOPICS[0]="default-$NOW"
-	CLEANUP[0]=""
+	CONFIG[0]=""
 
 	TOPICS[1]="compact-$NOW"
-	CLEANUP[1]="compact,compression.type=snappy"
+	CONFIG[1]="cleanup.policy=compact,compression.type=snappy"
 
 	KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh
 
-	# Loop through each array, validate that topic exists, and correct cleanup policy is set
+	# shellcheck disable=SC1091
+	source "/usr/bin/versions.sh"
+
+	# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+	# --bootstrap-server option with a random broker
+	if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+		CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)"
+	else
+		CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+	fi
+
+	# Loop through each array, validate that topic exists, and correct configuration is set
 	for i in "${!TOPICS[@]}"; do
 		TOPIC=${TOPICS[i]}
 
 		echo "Validating topic '$TOPIC'"
 
-		EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC")
-		POLICY=$(/opt/kafka/bin/kafka-configs.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --entity-type topics --entity-name "$TOPIC" --describe | grep 'Configs for topic' | awk -F'cleanup.policy=' '{print $2}')
+		# shellcheck disable=SC2086
+		EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC")
+		# shellcheck disable=SC2086
+		ACTUAL_CONFIG=$(/opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe \
+			| cut -d'{' -f1 \
+			| grep -oE '(compression.type|cleanup.policy)=([^ ,]+)' \
+			| sort \
+			| tr '\n' ',' \
+			| sed 's/,$//')
 
-		RESULT="$EXISTS:$POLICY"
-		EXPECTED="$TOPIC:${CLEANUP[i]}"
+		RESULT="$EXISTS:$ACTUAL_CONFIG"
+		EXPECTED="$TOPIC:${CONFIG[i]}"
 
 		if [[ "$RESULT" != "$EXPECTED" ]]; then
-			echo "$TOPIC topic not configured correctly: '$RESULT'"
+			echo "$TOPIC topic not configured correctly:"
+			echo "    Actual: '$RESULT'"
+			echo "  Expected: '$EXPECTED'"
 			return 1
 		fi
 	done
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 0aa8ad9b..05a7a453 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -49,7 +49,7 @@ services:
     image: confluentinc/cp-kafkacat:5.0.0
     environment:
        - BROKER_LIST
-       - KAFKA_VERSION=${KAFKA_VERSION-2.7.1}
+       - KAFKA_VERSION=${KAFKA_VERSION-3.0.0}
     volumes:
       - .:/tests
     working_dir: /tests
diff --git a/test/runAllTests.sh b/test/runAllTests.sh
index a575b21f..da81e67d 100755
--- a/test/runAllTests.sh
+++ b/test/runAllTests.sh
@@ -7,12 +7,12 @@ echo "BROKER_LIST=$BROKER_LIST"
 
 runAll() {
   # Tests that require kafka
-  docker-compose run --rm kafkatest
+  docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkatest
 
   RESULT=$?
   if [[ $RESULT -eq 0 ]]; then
     # Tests that require kafkacat
-    docker-compose run --rm kafkacattest
+    docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkacattest
     RESULT=$?
   fi
 

From a576cbf51e056be1140f3e3506a8ed44373b6965 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 12 May 2022 12:35:25 +0200
Subject: [PATCH 02/34] make test independent on how sort works

---
 test/0.10/test.create-topics.kafka.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh
index 694a320d..c6b6b4f7 100755
--- a/test/0.10/test.create-topics.kafka.sh
+++ b/test/0.10/test.create-topics.kafka.sh
@@ -11,7 +11,9 @@ testCreateTopics() {
 	CONFIG[0]=""
 
 	TOPICS[1]="compact-$NOW"
-	CLEANUP[1]="compression.type=snappy,cleanup.policy=compact"
+	CONFIG[1]=$(echo -e "compression.type=snappy\ncleanup.policy=compact" | sort \
+			| tr '\n' ',' \
+			| sed 's/,$//')
 
 	KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh
 

From f267754dfd5f5f12a1ace0383846b5d36f1fb03b Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 12 May 2022 17:23:05 +0200
Subject: [PATCH 03/34] add support for kafka 3.1.0

---
 .travis.yml                                   |  4 +-
 CHANGELOG.md                                  |  4 +-
 Dockerfile                                    |  2 +-
 start-kafka.sh                                | 48 ++++++++++++-------
 .../test.start-kafka-advertised-host.kafka.sh | 10 +++-
 ...est.start-kafka-bug-312-kafka-env.kafka.sh | 10 +++-
 ...st.start-kafka-bug-313-kafka-opts.kafka.sh | 10 +++-
 test/0.0/test.start-kafka-host-name.kafka.sh  | 10 +++-
 .../test.start-kafka-log4j-config.kafka.sh    | 12 +++--
 test/0.0/test.start-kafka-restart.kafka.sh    | 10 +++-
 test/Readme.md                                |  2 +-
 test/docker-compose.yml                       |  3 +-
 12 files changed, 93 insertions(+), 32 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e992d4c3..0abc081c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ services:
 # This version will be also tagged as 'latest'
 env:
   global:
-    - LATEST="2.13-3.0.0"
+    - LATEST="2.13-3.1.0"
 
 # Build recommended versions based on: http://kafka.apache.org/downloads
 matrix:
@@ -30,7 +30,7 @@ matrix:
   - scala: 2.13
     env: KAFKA_VERSION=2.8.1
   - scala: 2.13
-    env: KAFKA_VERSION=3.0.0    
+    env: KAFKA_VERSION=3.1.0
 
 # Upgrade Docker Engine so we can use buildx
 before_install:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d5916154..32ae4da7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,10 +2,10 @@ Changelog
 =========
 
 Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date.
-09-Apr-2022
+12-May-2022
 -----------
 
-- Add support for Kafka `3.0.0`
+- Add support for Kafka `3.1.0`
 09-Apr-2022
 ----------
 
diff --git a/Dockerfile b/Dockerfile
index f01a5552..54a022a4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
 FROM openjdk:11-jre-slim
 
-ARG kafka_version=3.0.0
+ARG kafka_version=3.1.0
 ARG scala_version=2.13
 ARG vcs_ref=unspecified
 ARG build_date=unspecified
diff --git a/start-kafka.sh b/start-kafka.sh
index fb56deb5..6c2d7ac5 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -13,9 +13,24 @@ fi
 # Store original IFS config, so we can restore it at various stages
 ORIG_IFS=$IFS
 
-if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
-    echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"
-    exit 1
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    if [[ -z "BROKER_LIST" ]]; then
+        echo "ERROR: missing mandatory config: BROKER_LIST"
+        exit 1
+    fi
+    if [[ -v KAFKA_ADVERTISED_HOST_NAME ]]; then
+        echo "ERROR: KAFKA_ADVERTISED_HOST_NAME is removed as of kafka 3, remove KAFKA_ADVERTISED_HOST_NAME=$KAFKA_ADVERTISED_HOST_NAME from your config"
+        exit 1
+    fi
+        if [[ -v KAFKA_ADVERTISED_PORT ]]; then
+        echo "ERROR: KAFKA_ADVERTISED_PORT is removed as of kafka 3, remove KAFKA_ADVERTISED_PORT=$KAFKA_ADVERTISED_PORT from your config"
+        exit 1
+    fi
+else
+    if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
+        echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"
+        exit 1
+    fi
 fi
 
 if [[ -z "$KAFKA_PORT" ]]; then
@@ -84,21 +99,22 @@ if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then
 fi
 
 # Try and configure minimal settings or exit with error if there isn't enough information
-if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then
-    if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then
-        echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS"
-        exit 1
-    elif [[ -z "$HOSTNAME_VALUE" ]]; then
-        echo "ERROR: No listener or advertised hostname configuration provided in environment."
-        echo "       Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME"
-        exit 1
-    fi
+if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+    if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then
+        if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+            echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS"
+            exit 1
+        elif [[ -z "$HOSTNAME_VALUE" ]]; then
+            echo "ERROR: No listener or advertised hostname configuration provided in environment."
+            echo "       Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME"
+            exit 1
+        fi
 
-    # Maintain existing behaviour
-    # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined.
-    export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE"
+        # Maintain existing behaviour
+        # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined.
+        export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE"
+    fi
 fi
-
 # `advertised.port` and `advertised.host.name` are removed with Kafka 3.0.0
 # See: https://github.com/apache/kafka/pull/10872
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
diff --git a/test/0.0/test.start-kafka-advertised-host.kafka.sh b/test/0.0/test.start-kafka-advertised-host.kafka.sh
index 1791d60b..fee8ce23 100755
--- a/test/0.0/test.start-kafka-advertised-host.kafka.sh
+++ b/test/0.0/test.start-kafka-advertised-host.kafka.sh
@@ -16,5 +16,13 @@ testAdvertisedHost() {
 	assertAbsent 'advertised.listeners'
 	assertAbsent 'listeners'
 }
+source "/usr/bin/versions.sh"
 
-testAdvertisedHost
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    echo "with kafka from version 3.0.0 'advertised.port' and 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "See: https://github.com/apache/kafka/pull/10872"
+else
+    testAdvertisedHost
+fi
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
index 5f0167c3..e3ba9c1b 100755
--- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
@@ -21,5 +21,13 @@ testKafkaEnv() {
 
     echo " > Set KAFKA_OPTS=$KAFKA_OPTS"
 }
+source "/usr/bin/versions.sh"
 
-testKafkaEnv
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "See: https://github.com/apache/kafka/pull/10872"
+else
+    testKafkaEnv
+fi
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
index df7b4018..c854bc6c 100755
--- a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
@@ -19,5 +19,13 @@ testKafkaOpts() {
 
     echo "  > Set KAFKA_OPTS=$KAFKA_OPTS"
 }
+source "/usr/bin/versions.sh"
 
-testKafkaOpts
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "See: https://github.com/apache/kafka/pull/10872"
+else
+    testKafkaOpts
+fi
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-host-name.kafka.sh b/test/0.0/test.start-kafka-host-name.kafka.sh
index 08dc2113..d0f10512 100755
--- a/test/0.0/test.start-kafka-host-name.kafka.sh
+++ b/test/0.0/test.start-kafka-host-name.kafka.sh
@@ -14,5 +14,13 @@ testHostnameCommand() {
 	assertAbsent 'advertised.listeners'
 	assertAbsent 'listeners'
 }
+source "/usr/bin/versions.sh"
 
-testHostnameCommand
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "See: https://github.com/apache/kafka/pull/10872"
+else
+    testHostnameCommand
+fi
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-log4j-config.kafka.sh b/test/0.0/test.start-kafka-log4j-config.kafka.sh
index da4ff28e..5f67997a 100755
--- a/test/0.0/test.start-kafka-log4j-config.kafka.sh
+++ b/test/0.0/test.start-kafka-log4j-config.kafka.sh
@@ -4,7 +4,14 @@ source test.functions
 
 testLog4jConfig() {
 	# Given Log4j overrides are provided
-	export KAFKA_ADVERTISED_HOST_NAME="testhost"
+	source "/usr/bin/versions.sh"
+
+    # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+    # --bootstrap-server option with a random broker
+    if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+        export KAFKA_ADVERTISED_HOST_NAME="testhost"
+    fi
+
 	export LOG4J_LOGGER_KAFKA=DEBUG
 
 	# When the script is invoked
@@ -13,5 +20,4 @@ testLog4jConfig() {
 	# Then the configuration file is correct
 	assertExpectedLog4jConfig "log4j.logger.kafka=DEBUG"
 }
-
-testLog4jConfig
+testLog4jConfig
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-restart.kafka.sh b/test/0.0/test.start-kafka-restart.kafka.sh
index 68a25598..e719e709 100755
--- a/test/0.0/test.start-kafka-restart.kafka.sh
+++ b/test/0.0/test.start-kafka-restart.kafka.sh
@@ -14,5 +14,13 @@ testRestart() {
 	assertExpectedConfig 'advertised.host.name=testhost'
 	assertAbsent 'listeners'
 }
+source "/usr/bin/versions.sh"
 
-testRestart
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option with a random broker
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "See: https://github.com/apache/kafka/pull/10872"
+else
+    testRestart
+fi
\ No newline at end of file
diff --git a/test/Readme.md b/test/Readme.md
index cdc72c19..1b9d5d0a 100644
--- a/test/Readme.md
+++ b/test/Readme.md
@@ -18,7 +18,7 @@ Run selected tests
 ### Kafka
 
 ```
-docker-compose run --rm kafkatest <testname pattern>
+docker-compose run -e BROKER_LIST=$(../internal-broker-list.sh) --rm kafkatest <testname pattern>
 ```
 
 ### Kafkacat
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 05a7a453..72db766b 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -48,8 +48,7 @@ services:
   kafkacattest:
     image: confluentinc/cp-kafkacat:5.0.0
     environment:
-       - BROKER_LIST
-       - KAFKA_VERSION=${KAFKA_VERSION-3.0.0}
+       - KAFKA_VERSION=${KAFKA_VERSION-3.1.0}
     volumes:
       - .:/tests
     working_dir: /tests

From 5d7a593c41f0b4892a198b95c1a8a31b93cf41fc Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 18 May 2022 14:36:07 +0200
Subject: [PATCH 04/34] add newer and better versions

---
 .travis.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0abc081c..b16a66cc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ services:
 # This version will be also tagged as 'latest'
 env:
   global:
-    - LATEST="2.13-3.1.0"
+    - LATEST="2.13-3.2.0"
 
 # Build recommended versions based on: http://kafka.apache.org/downloads
 matrix:
@@ -30,7 +30,11 @@ matrix:
   - scala: 2.13
     env: KAFKA_VERSION=2.8.1
   - scala: 2.13
-    env: KAFKA_VERSION=3.1.0
+    env: KAFKA_VERSION=3.0.1
+  - scala: 2.13
+    env: KAFKA_VERSION=3.1.1
+  - scala: 2.13
+    env: KAFKA_VERSION=3.2.0
 
 # Upgrade Docker Engine so we can use buildx
 before_install:

From 8be81b0f58c5797248d9bf0b7111dd43538498fe Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 18 May 2022 14:42:41 +0200
Subject: [PATCH 05/34] make sure user removes obsolete config

---
 start-kafka.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/start-kafka.sh b/start-kafka.sh
index 6c2d7ac5..bf7f4a12 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -22,10 +22,18 @@ if [[ "$MAJOR_VERSION" -ge "3" ]]; then
         echo "ERROR: KAFKA_ADVERTISED_HOST_NAME is removed as of kafka 3, remove KAFKA_ADVERTISED_HOST_NAME=$KAFKA_ADVERTISED_HOST_NAME from your config"
         exit 1
     fi
-        if [[ -v KAFKA_ADVERTISED_PORT ]]; then
+    if [[ -v KAFKA_ADVERTISED_PORT ]]; then
         echo "ERROR: KAFKA_ADVERTISED_PORT is removed as of kafka 3, remove KAFKA_ADVERTISED_PORT=$KAFKA_ADVERTISED_PORT from your config"
         exit 1
     fi
+    if [[ -v KAFKA_PORT ]]; then
+        echo "ERROR: KAFKA_PORT is removed as of kafka 3, remove KAFKA_PORT=$KAFKA_PORT from your config"
+        exit 1
+    fi
+    if [[ -v KAFKA_HOST_NAME ]]; then
+        echo "ERROR: KAFKA_HOST_NAME is removed as of kafka 3, remove KAFKA_HOST_NAME=$KAFKA_HOST_NAME from your config"
+        exit 1
+    fi
 else
     if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
         echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"

From ecef794e94943c7e0bce5730bea824ca1cc57d84 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 18 May 2022 15:16:38 +0200
Subject: [PATCH 06/34] support 3.x in compose

---
 README.md                        | 2 +-
 docker-compose-single-broker.yml | 3 ++-
 docker-compose.yml               | 4 ++--
 start-kafka.sh                   | 3 +++
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index ca0ce84a..723b584d 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,7 @@ For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to sp
 
 ## Advertised hostname
 
-You can configure the advertised hostname in different ways
+You can configure the advertised hostname in different ways (Note that as of kafka 3 KAFKA_ADVERTISED_HOST_NAME,KAFKA_ADVERTISED_PORT KAFKA_PORT and KAFKA_HOST_NAME is no longer supported)
 
 1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME```
 2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"```
diff --git a/docker-compose-single-broker.yml b/docker-compose-single-broker.yml
index 4d8e9f51..e6f45373 100644
--- a/docker-compose-single-broker.yml
+++ b/docker-compose-single-broker.yml
@@ -9,7 +9,8 @@ services:
     ports:
       - "9092:9092"
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
       KAFKA_CREATE_TOPICS: "test:1:1"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
index b4f77b38..73e0d591 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -12,8 +12,8 @@ services:
       - "9092"
     environment:
       DOCKER_API_VERSION: 1.22
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     restart: unless-stopped
diff --git a/start-kafka.sh b/start-kafka.sh
index bf7f4a12..dced64da 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -34,6 +34,9 @@ if [[ "$MAJOR_VERSION" -ge "3" ]]; then
         echo "ERROR: KAFKA_HOST_NAME is removed as of kafka 3, remove KAFKA_HOST_NAME=$KAFKA_HOST_NAME from your config"
         exit 1
     fi
+    if [[ -v KAFKA_ZOOKEEPER_CONNECT ]]; then
+        echo "WARNING: KAFKA_ZOOKEEPER_CONNECT is not used as of kafka 3, KAFKA_ZOOKEEPER_CONNECT=$KAFKA_ZOOKEEPER_CONNECT from your config has no effect"
+    fi
 else
     if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
         echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"

From c527cb54652080d5d0b4530a80bda10e8981edb6 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 18 May 2022 15:25:43 +0200
Subject: [PATCH 07/34] fix typo

---
 test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh  | 2 +-
 test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh | 2 +-
 test/0.0/test.start-kafka-host-name.kafka.sh          | 2 +-
 test/0.0/test.start-kafka-restart.kafka.sh            | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
index e3ba9c1b..deffc51d 100755
--- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
@@ -26,7 +26,7 @@ source "/usr/bin/versions.sh"
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option with a random broker
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
     echo "See: https://github.com/apache/kafka/pull/10872"
 else
     testKafkaEnv
diff --git a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
index c854bc6c..a0267c15 100755
--- a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
@@ -24,7 +24,7 @@ source "/usr/bin/versions.sh"
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option with a random broker
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
     echo "See: https://github.com/apache/kafka/pull/10872"
 else
     testKafkaOpts
diff --git a/test/0.0/test.start-kafka-host-name.kafka.sh b/test/0.0/test.start-kafka-host-name.kafka.sh
index d0f10512..5ee216ba 100755
--- a/test/0.0/test.start-kafka-host-name.kafka.sh
+++ b/test/0.0/test.start-kafka-host-name.kafka.sh
@@ -19,7 +19,7 @@ source "/usr/bin/versions.sh"
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option with a random broker
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
     echo "See: https://github.com/apache/kafka/pull/10872"
 else
     testHostnameCommand
diff --git a/test/0.0/test.start-kafka-restart.kafka.sh b/test/0.0/test.start-kafka-restart.kafka.sh
index e719e709..7fe54a1b 100755
--- a/test/0.0/test.start-kafka-restart.kafka.sh
+++ b/test/0.0/test.start-kafka-restart.kafka.sh
@@ -19,7 +19,7 @@ source "/usr/bin/versions.sh"
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option with a random broker
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this thes is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
     echo "See: https://github.com/apache/kafka/pull/10872"
 else
     testRestart

From 289836a78da24c9fa6626cb4bd4ebc2cd979e8e1 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 18 May 2022 15:25:58 +0200
Subject: [PATCH 08/34] fix missing $ (typo)

---
 start-kafka.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/start-kafka.sh b/start-kafka.sh
index dced64da..e59f6cc7 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -14,7 +14,7 @@ fi
 ORIG_IFS=$IFS
 
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    if [[ -z "BROKER_LIST" ]]; then
+    if [[ -z "$BROKER_LIST" ]]; then
         echo "ERROR: missing mandatory config: BROKER_LIST"
         exit 1
     fi

From 426638f548f97edb57d69854eafa43cde59db82b Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 19 May 2022 08:29:55 +0200
Subject: [PATCH 09/34] re add BROKER_LIST, propper default KAFKA_VERSION

---
 test/docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 72db766b..577f7216 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -48,7 +48,8 @@ services:
   kafkacattest:
     image: confluentinc/cp-kafkacat:5.0.0
     environment:
-       - KAFKA_VERSION=${KAFKA_VERSION-3.1.0}
+       - BROKER_LIST
+       - KAFKA_VERSION=${KAFKA_VERSION-3.2.0}
     volumes:
       - .:/tests
     working_dir: /tests

From 823e024d2e7228ff009d161c8ffb1785981369ea Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 19 May 2022 08:37:58 +0200
Subject: [PATCH 10/34] make shell test happy

---
 test/0.0/test.start-kafka-advertised-host.kafka.sh    | 1 +
 test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh  | 1 +
 test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh | 1 +
 test/0.0/test.start-kafka-host-name.kafka.sh          | 1 +
 test/0.0/test.start-kafka-log4j-config.kafka.sh       | 1 +
 test/0.0/test.start-kafka-restart.kafka.sh            | 1 +
 6 files changed, 6 insertions(+)

diff --git a/test/0.0/test.start-kafka-advertised-host.kafka.sh b/test/0.0/test.start-kafka-advertised-host.kafka.sh
index fee8ce23..6c683755 100755
--- a/test/0.0/test.start-kafka-advertised-host.kafka.sh
+++ b/test/0.0/test.start-kafka-advertised-host.kafka.sh
@@ -16,6 +16,7 @@ testAdvertisedHost() {
 	assertAbsent 'advertised.listeners'
 	assertAbsent 'listeners'
 }
+# shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
index deffc51d..7e21c713 100755
--- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
@@ -21,6 +21,7 @@ testKafkaEnv() {
 
     echo " > Set KAFKA_OPTS=$KAFKA_OPTS"
 }
+# shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
diff --git a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
index a0267c15..cf1c8f4c 100755
--- a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
@@ -19,6 +19,7 @@ testKafkaOpts() {
 
     echo "  > Set KAFKA_OPTS=$KAFKA_OPTS"
 }
+# shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
diff --git a/test/0.0/test.start-kafka-host-name.kafka.sh b/test/0.0/test.start-kafka-host-name.kafka.sh
index 5ee216ba..dfca349d 100755
--- a/test/0.0/test.start-kafka-host-name.kafka.sh
+++ b/test/0.0/test.start-kafka-host-name.kafka.sh
@@ -14,6 +14,7 @@ testHostnameCommand() {
 	assertAbsent 'advertised.listeners'
 	assertAbsent 'listeners'
 }
+# shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
diff --git a/test/0.0/test.start-kafka-log4j-config.kafka.sh b/test/0.0/test.start-kafka-log4j-config.kafka.sh
index 5f67997a..58794a5f 100755
--- a/test/0.0/test.start-kafka-log4j-config.kafka.sh
+++ b/test/0.0/test.start-kafka-log4j-config.kafka.sh
@@ -4,6 +4,7 @@ source test.functions
 
 testLog4jConfig() {
 	# Given Log4j overrides are provided
+	# shellcheck disable=SC1091
 	source "/usr/bin/versions.sh"
 
     # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
diff --git a/test/0.0/test.start-kafka-restart.kafka.sh b/test/0.0/test.start-kafka-restart.kafka.sh
index 7fe54a1b..6143307d 100755
--- a/test/0.0/test.start-kafka-restart.kafka.sh
+++ b/test/0.0/test.start-kafka-restart.kafka.sh
@@ -14,6 +14,7 @@ testRestart() {
 	assertExpectedConfig 'advertised.host.name=testhost'
 	assertAbsent 'listeners'
 }
+# shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the

From c3c455f7368b386a7bd31265565b83ea08a9415d Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 19 May 2022 09:26:40 +0200
Subject: [PATCH 11/34] propper default

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 54a022a4..f77de0c7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
 FROM openjdk:11-jre-slim
 
-ARG kafka_version=3.1.0
+ARG kafka_version=3.2.0
 ARG scala_version=2.13
 ARG vcs_ref=unspecified
 ARG build_date=unspecified

From 57af850f2de5d89d98e383744443ce900c25ced5 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 19 May 2022 09:26:59 +0200
Subject: [PATCH 12/34] do not require BROKER_LIST

---
 start-kafka.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/start-kafka.sh b/start-kafka.sh
index e59f6cc7..76f6245a 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -14,10 +14,6 @@ fi
 ORIG_IFS=$IFS
 
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    if [[ -z "$BROKER_LIST" ]]; then
-        echo "ERROR: missing mandatory config: BROKER_LIST"
-        exit 1
-    fi
     if [[ -v KAFKA_ADVERTISED_HOST_NAME ]]; then
         echo "ERROR: KAFKA_ADVERTISED_HOST_NAME is removed as of kafka 3, remove KAFKA_ADVERTISED_HOST_NAME=$KAFKA_ADVERTISED_HOST_NAME from your config"
         exit 1

From 750ebb18fdf66f0ea97984a263aad8ac1eabca0e Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 19 May 2022 15:05:37 +0200
Subject: [PATCH 13/34] refactor removed arguments for kafka

---
 .travis.yml                    | 47 +++++++++++++++++++++++++---------
 start-kafka.sh                 |  9 +++----
 test/docker-compose-kafka.yml  | 23 +++++++++++++++++
 test/docker-compose-kafka3.yml | 26 +++++++++++++++++++
 test/docker-compose.yml        |  8 ------
 5 files changed, 87 insertions(+), 26 deletions(-)
 create mode 100644 test/docker-compose-kafka.yml
 create mode 100644 test/docker-compose-kafka3.yml

diff --git a/.travis.yml b/.travis.yml
index b16a66cc..3253c94f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,27 +14,50 @@ env:
 matrix:
   include:
   - scala: 2.12
-    env: KAFKA_VERSION=2.1.1
+    env:
+      - KAFKA_VERSION=2.1.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.2.2
+    env:
+      - KAFKA_VERSION=2.2.2
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.3.1
+    env:
+      - KAFKA_VERSION=2.3.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.4.1
+    env:
+      - KAFKA_VERSION=2.4.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.12
-    env: KAFKA_VERSION=2.5.1
+    env:
+      - KAFKA_VERSION=2.5.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.6.3
+    env:
+      - KAFKA_VERSION=2.6.3
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.7.2
+    env:
+      - KAFKA_VERSION=2.7.2
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=2.8.1
+    env:
+      - KAFKA_VERSION=2.8.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml
   - scala: 2.13
-    env: KAFKA_VERSION=3.0.1
+    env:
+      - KAFKA_VERSION=3.0.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
   - scala: 2.13
-    env: KAFKA_VERSION=3.1.1
+    env:
+      - KAFKA_VERSION=3.1.1
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
   - scala: 2.13
-    env: KAFKA_VERSION=3.2.0
+    env:
+      - KAFKA_VERSION=3.2.0
+      - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml
+
 
 # Upgrade Docker Engine so we can use buildx
 before_install:
@@ -66,7 +89,7 @@ install:
   - docker pull confluentinc/cp-kafkacat
 
 before_script:
-  - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2
+  - docker-compose -f test/docker-compose.yml -f ${EXTRA_COMPOSE_FILE} up -d zookeeper kafka_1 kafka_2
 
 script:
   # Shellcheck main source files
diff --git a/start-kafka.sh b/start-kafka.sh
index 76f6245a..8812c0c1 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -30,19 +30,16 @@ if [[ "$MAJOR_VERSION" -ge "3" ]]; then
         echo "ERROR: KAFKA_HOST_NAME is removed as of kafka 3, remove KAFKA_HOST_NAME=$KAFKA_HOST_NAME from your config"
         exit 1
     fi
-    if [[ -v KAFKA_ZOOKEEPER_CONNECT ]]; then
-        echo "WARNING: KAFKA_ZOOKEEPER_CONNECT is not used as of kafka 3, KAFKA_ZOOKEEPER_CONNECT=$KAFKA_ZOOKEEPER_CONNECT from your config has no effect"
-    fi
 else
     if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
         echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT"
         exit 1
     fi
+    if [[ -z "$KAFKA_PORT" ]]; then
+        export KAFKA_PORT=9092
+    fi
 fi
 
-if [[ -z "$KAFKA_PORT" ]]; then
-    export KAFKA_PORT=9092
-fi
 
 create-topics.sh &
 unset KAFKA_CREATE_TOPICS
diff --git a/test/docker-compose-kafka.yml b/test/docker-compose-kafka.yml
new file mode 100644
index 00000000..6786c860
--- /dev/null
+++ b/test/docker-compose-kafka.yml
@@ -0,0 +1,23 @@
+version: '2.1'
+
+x-kafka-environment-defaults: &kafka-environment-defaults
+  HOSTNAME_COMMAND: "echo $$(hostname)"
+  KAFKA_ADVERTISED_PORT: 9092
+  KAFKA_PORT: 9092
+  KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+
+services:
+  kafka_1:
+    environment:
+      <<: *kafka-environment-defaults
+
+  kafka_2:
+    environment:
+      <<: *kafka-environment-defaults
+
+  kafkatest:
+    environment:
+      <<: *kafka-environment-defaults
+      HOSTNAME_COMMAND: "echo $$(hostname)"
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
\ No newline at end of file
diff --git a/test/docker-compose-kafka3.yml b/test/docker-compose-kafka3.yml
new file mode 100644
index 00000000..35692049
--- /dev/null
+++ b/test/docker-compose-kafka3.yml
@@ -0,0 +1,26 @@
+version: '2.1'
+
+x-kafka-environment-defaults: &kafka-environment-defaults
+  HOSTNAME_COMMAND: "echo $$(hostname)"
+  KAFKA_LISTENERS: PLAINTEXT://:9092
+  KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+
+services:
+  kafka_1:
+    environment:
+      <<: *kafka-environment-defaults
+      KAFKA_BROKER_ID: 1
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_1:9092
+
+  kafka_2:
+    environment:
+      <<: *kafka-environment-defaults
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_2:9092
+      KAFKA_BROKER_ID: 2
+
+  kafkatest:
+    environment:
+      <<: *kafka-environment-defaults
+      HOSTNAME_COMMAND: "echo $$(hostname)"
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
\ No newline at end of file
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 577f7216..483c97a4 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -7,12 +7,6 @@ x-kafka-defaults: &kafka-defaults
   volumes:
     - /var/run/docker.sock:/var/run/docker.sock
 
-x-kafka-environment-defaults: &kafka-environment-defaults
-  HOSTNAME_COMMAND: "echo $$(hostname)"
-  KAFKA_ADVERTISED_PORT: 9092
-  KAFKA_PORT: 9092
-  KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-
 services:
   zookeeper:
     image: wurstmeister/zookeeper
@@ -22,13 +16,11 @@ services:
     <<: *kafka-defaults
     container_name: test_kafka_1
     environment:
-      <<: *kafka-environment-defaults
       KAFKA_BROKER_ID: 1
   kafka_2:
     <<: *kafka-defaults
     container_name: test_kafka_2
     environment:
-      <<: *kafka-environment-defaults
       KAFKA_BROKER_ID: 2
 
   kafkatest:

From 01fa192a3bf8d7204cb37568df363bfa389d0607 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 09:43:51 +0200
Subject: [PATCH 14/34] start kafka with propper options

---
 test/Readme.md                               |  5 ++++-
 test/docker-compose-kafka3.yml               |  2 --
 test/docker-compose.yml                      |  1 -
 test/scenarios/jmx/docker-compose-kafka.yml  | 11 +++++++++++
 test/scenarios/jmx/docker-compose-kafka3.yml |  6 ++++++
 test/scenarios/jmx/docker-compose.yml        |  4 ----
 test/scenarios/runJmxScenario.sh             | 10 +++++++++-
 7 files changed, 30 insertions(+), 9 deletions(-)
 create mode 100644 test/scenarios/jmx/docker-compose-kafka.yml
 create mode 100644 test/scenarios/jmx/docker-compose-kafka3.yml

diff --git a/test/Readme.md b/test/Readme.md
index 1b9d5d0a..acc4f7f9 100644
--- a/test/Readme.md
+++ b/test/Readme.md
@@ -8,7 +8,10 @@ To execute
 
 ```
 cd test
-docker-compose up -d zookeeper kafka_1 kafka_2
+# if kafka veresion < 3
+docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka_1 kafka_2
+# if kafka veresion >= 3
+docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka_1 kafka_2
 ./runAllTests.sh
 ```
 
diff --git a/test/docker-compose-kafka3.yml b/test/docker-compose-kafka3.yml
index 35692049..2fa0a743 100644
--- a/test/docker-compose-kafka3.yml
+++ b/test/docker-compose-kafka3.yml
@@ -9,14 +9,12 @@ services:
   kafka_1:
     environment:
       <<: *kafka-environment-defaults
-      KAFKA_BROKER_ID: 1
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_1:9092
 
   kafka_2:
     environment:
       <<: *kafka-environment-defaults
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_2:9092
-      KAFKA_BROKER_ID: 2
 
   kafkatest:
     environment:
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 483c97a4..78d7f35d 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -26,7 +26,6 @@ services:
   kafkatest:
     image: wurstmeister/kafka
     environment:
-      KAFKA_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:
       - .:/tests
diff --git a/test/scenarios/jmx/docker-compose-kafka.yml b/test/scenarios/jmx/docker-compose-kafka.yml
new file mode 100644
index 00000000..7acb91a8
--- /dev/null
+++ b/test/scenarios/jmx/docker-compose-kafka.yml
@@ -0,0 +1,11 @@
+version: '2'
+services:
+  kafka:
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_PORT: 9092
+
+  test:
+    environment:
+      KAFKA_PORT: 9092
diff --git a/test/scenarios/jmx/docker-compose-kafka3.yml b/test/scenarios/jmx/docker-compose-kafka3.yml
new file mode 100644
index 00000000..14b56722
--- /dev/null
+++ b/test/scenarios/jmx/docker-compose-kafka3.yml
@@ -0,0 +1,6 @@
+version: '2'
+services:
+  kafka:
+    environment:
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_1:9092
\ No newline at end of file
diff --git a/test/scenarios/jmx/docker-compose.yml b/test/scenarios/jmx/docker-compose.yml
index aba78bf2..0b9e578d 100644
--- a/test/scenarios/jmx/docker-compose.yml
+++ b/test/scenarios/jmx/docker-compose.yml
@@ -10,9 +10,6 @@ services:
       - "9092"
       - "1099"
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: kafka
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_PORT: 9092
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099"
@@ -31,7 +28,6 @@ services:
   test:
     image: wurstmeister/kafka
     environment:
-      KAFKA_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:
       - .:/scenario
diff --git a/test/scenarios/runJmxScenario.sh b/test/scenarios/runJmxScenario.sh
index e99b8851..1be24b42 100755
--- a/test/scenarios/runJmxScenario.sh
+++ b/test/scenarios/runJmxScenario.sh
@@ -3,7 +3,15 @@
 set -e -o pipefail
 
 pushd jmx
-docker-compose up -d zookeeper kafka jmxexporter
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+# since 3.0.0 a few options has been disabled, start kafka with the propper options
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka jmxexporter
+else
+    docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka jmxexporter
+fi
+
 docker-compose run --rm test
 docker-compose stop
 popd

From fe193118579d9d0cc702ea28e2a4c525733b50a0 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 09:52:41 +0200
Subject: [PATCH 15/34] supply entire list of bootstrap servers

---
 create-topics.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/create-topics.sh b/create-topics.sh
index 3ccbe194..125c295a 100755
--- a/create-topics.sh
+++ b/create-topics.sh
@@ -36,7 +36,7 @@ fi
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option with a random broker
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)"
+    CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}"
 else
     CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
 fi

From 8a8daf4404e342b551ad389e2352244e0ba014d1 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 12:46:36 +0200
Subject: [PATCH 16/34] various test cleanup to support removal of
 KAFKA_ADVERTISED_HOST_NAME KAFKA_ADVERTISED_PORT KAFKA_PORT and
 KAFKA_HOST_NAME

---
 CHANGELOG.md                                  |  4 +--
 create-topics.sh                              |  2 +-
 start-kafka.sh                                | 28 ++++++++-------
 .../test.start-kafka-advertised-host.kafka.sh |  2 +-
 ...est.start-kafka-bug-312-kafka-env.kafka.sh | 20 +++++------
 ...st.start-kafka-bug-313-kafka-opts.kafka.sh | 22 ++++++------
 test/0.0/test.start-kafka-host-name.kafka.sh  | 28 +++++++--------
 .../test.start-kafka-log4j-config.kafka.sh    |  9 +++--
 test/0.0/test.start-kafka-restart.kafka.sh    | 34 ++++++++++---------
 9 files changed, 73 insertions(+), 76 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 32ae4da7..fa848a48 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,10 +2,10 @@ Changelog
 =========
 
 Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date.
-12-May-2022
+20-May-2022
 -----------
 
-- Add support for Kafka `3.1.0`
+- Add support for Kafka `3.0.1`,`3.1.1` and `3.2.0`
 09-Apr-2022
 ----------
 
diff --git a/create-topics.sh b/create-topics.sh
index 125c295a..60d5ccd5 100755
--- a/create-topics.sh
+++ b/create-topics.sh
@@ -34,7 +34,7 @@ if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSI
 fi
 
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-# --bootstrap-server option with a random broker
+# --bootstrap-server option.
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
     CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}"
 else
diff --git a/start-kafka.sh b/start-kafka.sh
index 8812c0c1..148fa577 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -44,12 +44,14 @@ fi
 create-topics.sh &
 unset KAFKA_CREATE_TOPICS
 
-if [[ -z "$KAFKA_ADVERTISED_PORT" && \
-  -z "$KAFKA_LISTENERS" && \
-  -z "$KAFKA_ADVERTISED_LISTENERS" && \
-  -S /var/run/docker.sock ]]; then
-    KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1)
-    export KAFKA_ADVERTISED_PORT
+if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+    if [[ -z "$KAFKA_ADVERTISED_PORT" && \
+      -z "$KAFKA_LISTENERS" && \
+      -z "$KAFKA_ADVERTISED_LISTENERS" && \
+      -S /var/run/docker.sock ]]; then
+        KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1)
+        export KAFKA_ADVERTISED_PORT
+    fi
 fi
 
 if [[ -z "$KAFKA_BROKER_ID" ]]; then
@@ -118,13 +120,13 @@ if [[ "$MAJOR_VERSION" -lt "3" ]]; then
         # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined.
         export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE"
     fi
-fi
-# `advertised.port` and `advertised.host.name` are removed with Kafka 3.0.0
-# See: https://github.com/apache/kafka/pull/10872
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    if [ -z "$KAFKA_ADVERTISED_LISTENERS" ]; then
-        if [ -n "${KAFKA_ADVERTISED_HOST_NAME}" ]; then
-            export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:${KAFKA_PORT}"
+else
+    if [[ "$HOSTNAME_VALUE" ]]; then
+        if [[ -z "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+            export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$HOSTNAME_VALUE:9092"
+            if [[ -z "$KAFKA_LISTENERS" ]]; then
+                export KAFKA_LISTENERS=PLAINTEXT://:9092
+            fi
         fi
     fi
 fi
diff --git a/test/0.0/test.start-kafka-advertised-host.kafka.sh b/test/0.0/test.start-kafka-advertised-host.kafka.sh
index 6c683755..427df650 100755
--- a/test/0.0/test.start-kafka-advertised-host.kafka.sh
+++ b/test/0.0/test.start-kafka-advertised-host.kafka.sh
@@ -22,7 +22,7 @@ source "/usr/bin/versions.sh"
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option with a random broker
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "with kafka from version 3.0.0 'advertised.port' and 'advertised.host.name' are removed with Kafka 3.0.0"
+    echo "with kafka from version 3.0.0 'advertised.port' and 'advertised.host.name' are removed, making this test obosolete"
     echo "See: https://github.com/apache/kafka/pull/10872"
 else
     testAdvertisedHost
diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
index 7e21c713..9c0e4a44 100755
--- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
@@ -4,7 +4,12 @@ source test.functions
 
 testKafkaEnv() {
     # Given required settings are provided
-    export KAFKA_ADVERTISED_HOST_NAME="testhost"
+    source "/usr/bin/versions.sh"
+
+    # since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed
+    if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+        export KAFKA_ADVERTISED_HOST_NAME="testhost"
+    fi
     export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf"
 
     # When the script is invoked
@@ -21,14 +26,5 @@ testKafkaEnv() {
 
     echo " > Set KAFKA_OPTS=$KAFKA_OPTS"
 }
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
-
-# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-# --bootstrap-server option with a random broker
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
-    echo "See: https://github.com/apache/kafka/pull/10872"
-else
-    testKafkaEnv
-fi
\ No newline at end of file
+
+testKafkaEnv
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
index cf1c8f4c..720c2f39 100755
--- a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh
@@ -3,8 +3,15 @@
 source test.functions
 
 testKafkaOpts() {
+    # shellcheck disable=SC1091
+    source "/usr/bin/versions.sh"
+
+    # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME
+    if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+        export KAFKA_ADVERTISED_HOST_NAME="testhost"
+    fi
     # Given required settings are provided
-    export KAFKA_ADVERTISED_HOST_NAME="testhost"
+
     # .. and a CUSTOM_INIT_SCRIPT with spaces
     export CUSTOM_INIT_SCRIPT="export KAFKA_OPTS=-Djava.security.auth.login.config=/kafka_server_jaas.conf"
 
@@ -19,14 +26,5 @@ testKafkaOpts() {
 
     echo "  > Set KAFKA_OPTS=$KAFKA_OPTS"
 }
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
-
-# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-# --bootstrap-server option with a random broker
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
-    echo "See: https://github.com/apache/kafka/pull/10872"
-else
-    testKafkaOpts
-fi
\ No newline at end of file
+
+testKafkaOpts
\ No newline at end of file
diff --git a/test/0.0/test.start-kafka-host-name.kafka.sh b/test/0.0/test.start-kafka-host-name.kafka.sh
index dfca349d..5aabf205 100755
--- a/test/0.0/test.start-kafka-host-name.kafka.sh
+++ b/test/0.0/test.start-kafka-host-name.kafka.sh
@@ -8,20 +8,20 @@ testHostnameCommand() {
 
 	# When the script is invoked
 	source "$START_KAFKA"
+	# shellcheck disable=SC1091
+	source "/usr/bin/versions.sh"
 
-	# Then the configuration uses the value from the command
-	assertExpectedConfig 'advertised.host.name=my-host'
-	assertAbsent 'advertised.listeners'
-	assertAbsent 'listeners'
+	# since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME
+	if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+	    # Then the configuration uses the value from the command
+		assertExpectedConfig 'advertised.listeners=PLAINTEXT://my-host:9092'
+		assertExpectedConfig 'listeners=PLAINTEXT://:9092'
+	else
+		# Then the configuration uses the value from the command
+		assertExpectedConfig 'advertised.host.name=my-host'
+		assertAbsent 'advertised.listeners'
+		assertAbsent 'listeners'
+	fi
 }
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
 
-# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-# --bootstrap-server option with a random broker
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
-    echo "See: https://github.com/apache/kafka/pull/10872"
-else
-    testHostnameCommand
-fi
\ No newline at end of file
+testHostnameCommand
diff --git a/test/0.0/test.start-kafka-log4j-config.kafka.sh b/test/0.0/test.start-kafka-log4j-config.kafka.sh
index 58794a5f..f1d68614 100755
--- a/test/0.0/test.start-kafka-log4j-config.kafka.sh
+++ b/test/0.0/test.start-kafka-log4j-config.kafka.sh
@@ -7,11 +7,10 @@ testLog4jConfig() {
 	# shellcheck disable=SC1091
 	source "/usr/bin/versions.sh"
 
-    # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-    # --bootstrap-server option with a random broker
-    if [[ "$MAJOR_VERSION" -lt "3" ]]; then
-        export KAFKA_ADVERTISED_HOST_NAME="testhost"
-    fi
+	# since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME
+	if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+		export KAFKA_ADVERTISED_HOST_NAME="testhost"
+	fi
 
 	export LOG4J_LOGGER_KAFKA=DEBUG
 
diff --git a/test/0.0/test.start-kafka-restart.kafka.sh b/test/0.0/test.start-kafka-restart.kafka.sh
index 6143307d..6d526c4b 100755
--- a/test/0.0/test.start-kafka-restart.kafka.sh
+++ b/test/0.0/test.start-kafka-restart.kafka.sh
@@ -3,25 +3,27 @@
 source test.functions
 
 testRestart() {
-	# Given a hostname is provided
-	export KAFKA_ADVERTISED_HOST_NAME="testhost"
+	# shellcheck disable=SC1091
+	source "/usr/bin/versions.sh"
+	# since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed
+	if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+		# Given a hostname is provided
+		export KAFKA_ADVERTISED_HOST_NAME="testhost"
+	else
+        export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://testhost:9092"
+	fi
 
 	# When the container is restarted (Script invoked multiple times)
 	source "$START_KAFKA"
 	source "$START_KAFKA"
 
-	# Then the configuration file only has one instance of the config
-	assertExpectedConfig 'advertised.host.name=testhost'
-	assertAbsent 'listeners'
+	if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+		# Then the configuration file only has one instance of the config
+	    assertExpectedConfig 'advertised.host.name=testhost'
+	    assertAbsent 'listeners'
+	else
+	    assertExpectedConfig 'advertised.listeners=PLAINTEXT://testhost:9092'
+	    assertAbsent 'listeners'
+	fi
 }
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
-
-# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-# --bootstrap-server option with a random broker
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    echo "this test is obsolete with kafka from version 3.0.0 'advertised.host.name' are removed with Kafka 3.0.0"
-    echo "See: https://github.com/apache/kafka/pull/10872"
-else
-    testRestart
-fi
\ No newline at end of file
+testRestart
\ No newline at end of file

From 8914a834efd12c51e4d6e882477ac221db338ba9 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 13:03:33 +0200
Subject: [PATCH 17/34] fix shell check

---
 test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
index 9c0e4a44..59703095 100755
--- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
+++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh
@@ -3,14 +3,16 @@
 source test.functions
 
 testKafkaEnv() {
-    # Given required settings are provided
+    # shellcheck disable=SC1091
     source "/usr/bin/versions.sh"
 
+    # Given required settings are provided
+    export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf"
     # since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed
     if [[ "$MAJOR_VERSION" -lt "3" ]]; then
+
         export KAFKA_ADVERTISED_HOST_NAME="testhost"
     fi
-    export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf"
 
     # When the script is invoked
     source "$START_KAFKA"

From 724866b5ef6d8c2b3925b65239c6512551121306 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 13:19:34 +0200
Subject: [PATCH 18/34] make MAJOR_VERSION accessible for test scripts

---
 .travis.yml                      | 1 +
 test/scenarios/runJmxScenario.sh | 8 +++-----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 3253c94f..0b44cb91 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -94,6 +94,7 @@ before_script:
 script:
   # Shellcheck main source files
   - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
+  - source ./version.sh
   - cd test
   # Shellcheck the tests
   - shellcheck -x -e SC1090 -s bash *.sh **/*.sh
diff --git a/test/scenarios/runJmxScenario.sh b/test/scenarios/runJmxScenario.sh
index 1be24b42..1cd68310 100755
--- a/test/scenarios/runJmxScenario.sh
+++ b/test/scenarios/runJmxScenario.sh
@@ -3,13 +3,11 @@
 set -e -o pipefail
 
 pushd jmx
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
 # since 3.0.0 a few options has been disabled, start kafka with the propper options
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka jmxexporter
-else
+if [[ "$MAJOR_VERSION" -le "3" ]]; then
     docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka jmxexporter
+else
+    docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka jmxexporter
 fi
 
 docker-compose run --rm test

From a97e332f993262a4e50d1165af136d636591e978 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 13:56:37 +0200
Subject: [PATCH 19/34] propper naming of versions.sh

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 0b44cb91..1a3bc94e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -94,7 +94,7 @@ before_script:
 script:
   # Shellcheck main source files
   - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
-  - source ./version.sh
+  - source ./versions.sh
   - cd test
   # Shellcheck the tests
   - shellcheck -x -e SC1090 -s bash *.sh **/*.sh

From 8d80440462ab417d5f6c81e08bf007a7c3ac6ddc Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 14:16:24 +0200
Subject: [PATCH 20/34] fix selection of kafka args

---
 test/scenarios/runJmxScenario.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scenarios/runJmxScenario.sh b/test/scenarios/runJmxScenario.sh
index 1cd68310..773a32b0 100755
--- a/test/scenarios/runJmxScenario.sh
+++ b/test/scenarios/runJmxScenario.sh
@@ -4,7 +4,7 @@ set -e -o pipefail
 
 pushd jmx
 # since 3.0.0 a few options has been disabled, start kafka with the propper options
-if [[ "$MAJOR_VERSION" -le "3" ]]; then
+if [[ "$MAJOR_VERSION" -lt "3" ]]; then
     docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka jmxexporter
 else
     docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka jmxexporter

From 13933fdef7b18566b01c7fcef5286c998e893472 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Fri, 20 May 2022 19:30:49 +0200
Subject: [PATCH 21/34] revievers comment about wrong hostname, changing to
 propper name kafka

---
 test/scenarios/jmx/docker-compose-kafka3.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scenarios/jmx/docker-compose-kafka3.yml b/test/scenarios/jmx/docker-compose-kafka3.yml
index 14b56722..221d2462 100644
--- a/test/scenarios/jmx/docker-compose-kafka3.yml
+++ b/test/scenarios/jmx/docker-compose-kafka3.yml
@@ -3,4 +3,4 @@ services:
   kafka:
     environment:
       KAFKA_LISTENERS: PLAINTEXT://:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_1:9092
\ No newline at end of file
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
\ No newline at end of file

From 14f089d572ae9ef81f7279370f11733bc38ac295 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Mon, 23 May 2022 08:31:01 +0200
Subject: [PATCH 22/34] reviewers comment about not needing seperate test setup
 for previous versions

---
 .travis.yml                                  |  1 -
 test/scenarios/jmx/docker-compose-kafka.yml  | 11 -----------
 test/scenarios/jmx/docker-compose-kafka3.yml |  6 ------
 test/scenarios/jmx/docker-compose.yml        |  2 ++
 test/scenarios/runJmxScenario.sh             |  7 +------
 5 files changed, 3 insertions(+), 24 deletions(-)
 delete mode 100644 test/scenarios/jmx/docker-compose-kafka.yml
 delete mode 100644 test/scenarios/jmx/docker-compose-kafka3.yml

diff --git a/.travis.yml b/.travis.yml
index 1a3bc94e..3253c94f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -94,7 +94,6 @@ before_script:
 script:
   # Shellcheck main source files
   - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
-  - source ./versions.sh
   - cd test
   # Shellcheck the tests
   - shellcheck -x -e SC1090 -s bash *.sh **/*.sh
diff --git a/test/scenarios/jmx/docker-compose-kafka.yml b/test/scenarios/jmx/docker-compose-kafka.yml
deleted file mode 100644
index 7acb91a8..00000000
--- a/test/scenarios/jmx/docker-compose-kafka.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-version: '2'
-services:
-  kafka:
-    environment:
-      KAFKA_ADVERTISED_HOST_NAME: kafka
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_PORT: 9092
-
-  test:
-    environment:
-      KAFKA_PORT: 9092
diff --git a/test/scenarios/jmx/docker-compose-kafka3.yml b/test/scenarios/jmx/docker-compose-kafka3.yml
deleted file mode 100644
index 221d2462..00000000
--- a/test/scenarios/jmx/docker-compose-kafka3.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-version: '2'
-services:
-  kafka:
-    environment:
-      KAFKA_LISTENERS: PLAINTEXT://:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
\ No newline at end of file
diff --git a/test/scenarios/jmx/docker-compose.yml b/test/scenarios/jmx/docker-compose.yml
index 0b9e578d..b86f4ddf 100644
--- a/test/scenarios/jmx/docker-compose.yml
+++ b/test/scenarios/jmx/docker-compose.yml
@@ -12,6 +12,8 @@ services:
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
       KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099"
       JMX_PORT: 1099
     volumes:
diff --git a/test/scenarios/runJmxScenario.sh b/test/scenarios/runJmxScenario.sh
index 773a32b0..e161a0d5 100755
--- a/test/scenarios/runJmxScenario.sh
+++ b/test/scenarios/runJmxScenario.sh
@@ -3,12 +3,7 @@
 set -e -o pipefail
 
 pushd jmx
-# since 3.0.0 a few options has been disabled, start kafka with the propper options
-if [[ "$MAJOR_VERSION" -lt "3" ]]; then
-    docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka jmxexporter
-else
-    docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka jmxexporter
-fi
+docker-compose -f docker-compose.yml up -d zookeeper kafka jmxexporter
 
 docker-compose run --rm test
 docker-compose stop

From b3e54b05594edfa0971a1f992f11f559e65a3f93 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Mon, 23 May 2022 08:42:38 +0200
Subject: [PATCH 23/34] trigger build


From 704e9ab49bc731d826db3dd98ee6a23e95ad7e51 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Mon, 23 May 2022 12:34:58 +0200
Subject: [PATCH 24/34] no need to specify compose file

---
 test/scenarios/runJmxScenario.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scenarios/runJmxScenario.sh b/test/scenarios/runJmxScenario.sh
index e161a0d5..b5964786 100755
--- a/test/scenarios/runJmxScenario.sh
+++ b/test/scenarios/runJmxScenario.sh
@@ -3,7 +3,7 @@
 set -e -o pipefail
 
 pushd jmx
-docker-compose -f docker-compose.yml up -d zookeeper kafka jmxexporter
+docker-compose up -d zookeeper kafka jmxexporter
 
 docker-compose run --rm test
 docker-compose stop

From 0698081aadf3e071eea36f1f090a3eb94bde1496 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Mon, 23 May 2022 14:41:37 +0200
Subject: [PATCH 25/34] totally revert this file

---
 test/scenarios/runJmxScenario.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/scenarios/runJmxScenario.sh b/test/scenarios/runJmxScenario.sh
index b5964786..e99b8851 100755
--- a/test/scenarios/runJmxScenario.sh
+++ b/test/scenarios/runJmxScenario.sh
@@ -4,7 +4,6 @@ set -e -o pipefail
 
 pushd jmx
 docker-compose up -d zookeeper kafka jmxexporter
-
 docker-compose run --rm test
 docker-compose stop
 popd

From 58dfb8fecde4371c6b42d68c895d2c185ef642ad Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Wed, 8 Jun 2022 14:37:41 +0200
Subject: [PATCH 26/34] trigger build


From 8922faa1a5d73d40d683ca49bfdc589197aea65d Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 30 Jun 2022 11:30:31 +0200
Subject: [PATCH 27/34] typo in readme, we are in the test folder when
 executing this, this is alos where  the internal-broker-list.sh is

---
 test/Readme.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/Readme.md b/test/Readme.md
index acc4f7f9..ba50b714 100644
--- a/test/Readme.md
+++ b/test/Readme.md
@@ -21,7 +21,7 @@ Run selected tests
 ### Kafka
 
 ```
-docker-compose run -e BROKER_LIST=$(../internal-broker-list.sh) --rm kafkatest <testname pattern>
+docker-compose run -e BROKER_LIST=$(./internal-broker-list.sh) --rm kafkatest <testname pattern>
 ```
 
 ### Kafkacat

From 2abb7db448e4e06b8a4d1ae11b4de21fbb267d20 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 30 Jun 2022 11:32:26 +0200
Subject: [PATCH 28/34] examine KAFKA_LISTNERS to determine the port we are
 runing on, also reflect this in tests, where it was not propperly mocked

---
 create-topics.sh                                 | 16 ++++++++++++----
 .../test.create-topics-custom-separator.kafka.sh | 12 +++++++++++-
 test/0.10/test.create-topics.kafka.sh            | 14 +++++++++++---
 3 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/create-topics.sh b/create-topics.sh
index 60d5ccd5..1627d8e6 100755
--- a/create-topics.sh
+++ b/create-topics.sh
@@ -11,7 +11,17 @@ fi
 start_timeout_exceeded=false
 count=0
 step=10
-while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do
+
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    PORT=$(echo $KAFKA_LISTENERS | awk -F: '{print $3}' )
+else
+    PORT="$KAFKA_PORT"
+fi
+
+
+while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do
     echo "waiting for kafka to be ready"
     sleep $step;
     count=$((count + step))
@@ -27,8 +37,6 @@ if $start_timeout_exceeded; then
 fi
 
 # introduced in 0.10. In earlier versions, this will fail because the topic already exists.
-# shellcheck disable=SC1091
-source "/usr/bin/versions.sh"
 if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then
     KAFKA_0_10_OPTS="--if-not-exists"
 fi
@@ -36,7 +44,7 @@ fi
 # since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 # --bootstrap-server option.
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}"
+    CONNECT_OPTS="--bootstrap-server ${PORT}"
 else
     CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
 fi
diff --git a/test/0.0/test.create-topics-custom-separator.kafka.sh b/test/0.0/test.create-topics-custom-separator.kafka.sh
index 4ad32bd3..063d8e9a 100755
--- a/test/0.0/test.create-topics-custom-separator.kafka.sh
+++ b/test/0.0/test.create-topics-custom-separator.kafka.sh
@@ -48,8 +48,18 @@ testCreateTopicsCustomSeparator() {
 	return 0
 }
 
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+
+# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    PORT=$(echo $KAFKA_LISTENERS | awk -F: '{print $3}')
+else
+    PORT="$KAFKA_PORT"
+fi
+
 # mock the netstat call as made by the create-topics.sh script
-function netstat() { echo "1 2 3 :$KAFKA_PORT"; }
+function netstat() { echo "1 2 3 :$PORT"; }
 export -f netstat
 
 testCreateTopicsCustomSeparator
diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh
index c6b6b4f7..9752f777 100755
--- a/test/0.10/test.create-topics.kafka.sh
+++ b/test/0.10/test.create-topics.kafka.sh
@@ -17,8 +17,6 @@ testCreateTopics() {
 
 	KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh
 
-	# shellcheck disable=SC1091
-	source "/usr/bin/versions.sh"
 
 	# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
 	# --bootstrap-server option with a random broker
@@ -58,8 +56,18 @@ testCreateTopics() {
 	return 0
 }
 
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+
+# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS
+if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+    PORT=$(echo $KAFKA_LISTENERS | awk -F: '{print $3}')
+else
+    PORT="$KAFKA_PORT"
+fi
+
 # mock the netstat call as made by the create-topics.sh script
-function netstat() { echo "1 2 3 :$KAFKA_PORT"; }
+function netstat() { echo "1 2 3 :$PORT"; }
 export -f netstat
 
 testCreateTopics

From 001df7dbb541905286eccd3fd67a44b051502731 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 30 Jun 2022 12:16:07 +0200
Subject: [PATCH 29/34] Double quote to prevent globbing

---
 create-topics.sh                                      | 2 +-
 test/0.0/test.create-topics-custom-separator.kafka.sh | 2 +-
 test/0.10/test.create-topics.kafka.sh                 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/create-topics.sh b/create-topics.sh
index 1627d8e6..3805470f 100755
--- a/create-topics.sh
+++ b/create-topics.sh
@@ -15,7 +15,7 @@ step=10
 # shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    PORT=$(echo $KAFKA_LISTENERS | awk -F: '{print $3}' )
+    PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' )
 else
     PORT="$KAFKA_PORT"
 fi
diff --git a/test/0.0/test.create-topics-custom-separator.kafka.sh b/test/0.0/test.create-topics-custom-separator.kafka.sh
index 063d8e9a..c26e8131 100755
--- a/test/0.0/test.create-topics-custom-separator.kafka.sh
+++ b/test/0.0/test.create-topics-custom-separator.kafka.sh
@@ -53,7 +53,7 @@ source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    PORT=$(echo $KAFKA_LISTENERS | awk -F: '{print $3}')
+    PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}')
 else
     PORT="$KAFKA_PORT"
 fi
diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh
index 9752f777..c8a1a6a1 100755
--- a/test/0.10/test.create-topics.kafka.sh
+++ b/test/0.10/test.create-topics.kafka.sh
@@ -61,7 +61,7 @@ source "/usr/bin/versions.sh"
 
 # since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    PORT=$(echo $KAFKA_LISTENERS | awk -F: '{print $3}')
+    PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}')
 else
     PORT="$KAFKA_PORT"
 fi

From 722fc5c75f9db53aebf9d0b06e837ce838be4461 Mon Sep 17 00:00:00 2001
From: Frank Vissing <frank.vissing@se.com>
Date: Thu, 30 Jun 2022 13:25:44 +0200
Subject: [PATCH 30/34] make create topics work both for KAFKA_LISTNERS and
 BROKER_LIST

---
 create-topics.sh | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/create-topics.sh b/create-topics.sh
index 3805470f..21bc83fb 100755
--- a/create-topics.sh
+++ b/create-topics.sh
@@ -14,22 +14,31 @@ step=10
 
 # shellcheck disable=SC1091
 source "/usr/bin/versions.sh"
+# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+# --bootstrap-server option.
 if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' )
+    if [[ -v KAFKA_LISTENERS ]]; then
+        PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' )
+        CONNECT_OPTS="--bootstrap-server localhost:${PORT}"
+    else
+        CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}"
+    fi
 else
     PORT="$KAFKA_PORT"
+    CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
 fi
 
-
-while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do
-    echo "waiting for kafka to be ready"
-    sleep $step;
-    count=$((count + step))
-    if [ $count -gt $START_TIMEOUT ]; then
-        start_timeout_exceeded=true
-        break
-    fi
-done
+if [[ ! -v PORT ]]; then
+    while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do
+        echo "waiting for kafka to be ready"
+        sleep $step;
+        count=$((count + step))
+        if [ $count -gt $START_TIMEOUT ]; then
+            start_timeout_exceeded=true
+            break
+        fi
+    done
+fi
 
 if $start_timeout_exceeded; then
     echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
@@ -41,14 +50,6 @@ if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSI
     KAFKA_0_10_OPTS="--if-not-exists"
 fi
 
-# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
-# --bootstrap-server option.
-if [[ "$MAJOR_VERSION" -ge "3" ]]; then
-    CONNECT_OPTS="--bootstrap-server ${PORT}"
-else
-    CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
-fi
-
 # Expected format:
 #   name:partitions:replicas:cleanup.policy
 IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do

From ce8cb9a0ede157fbe3c41738430c8fef00205af8 Mon Sep 17 00:00:00 2001
From: Stephen Sorriaux <stephen.sorriaux@gmail.com>
Date: Fri, 10 Mar 2023 16:51:56 -0500
Subject: [PATCH 31/34] Add new 3.X images to GA

---
 .github/workflows/package.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index a3cfbb83..244715ab 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -8,7 +8,7 @@ on:
     branches: ["master"]
 
 env:
-  LATEST: "2.13-2.8.1"
+  LATEST: "2.13-3.2.0"
   IMAGE_DEST: "ssorriaux/kafka"
 
 jobs:
@@ -35,6 +35,12 @@ jobs:
           kafka_version: 2.7.2
         - scala: 2.13
           kafka_version: 2.8.1
+        - scala: 2.13
+          kafka_version: 3.0.1
+        - scala: 2.13
+          kafka_version: 3.1.1
+        - scala: 2.13
+          kafka_version: 3.2.0
 
     steps:
     - name: Checkout

From 554019b40f2ba28ae859f142225f0aaa90c6a937 Mon Sep 17 00:00:00 2001
From: Stephen Sorriaux <stephen.sorriaux@gmail.com>
Date: Fri, 10 Mar 2023 16:59:37 -0500
Subject: [PATCH 32/34] Use latest version + add 3.3.X and 3.4.X

---
 .github/workflows/package.yml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 244715ab..45743ab0 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -34,13 +34,17 @@ jobs:
         - scala: 2.13
           kafka_version: 2.7.2
         - scala: 2.13
-          kafka_version: 2.8.1
+          kafka_version: 2.8.2
         - scala: 2.13
-          kafka_version: 3.0.1
+          kafka_version: 3.0.2
         - scala: 2.13
-          kafka_version: 3.1.1
+          kafka_version: 3.1.2
         - scala: 2.13
-          kafka_version: 3.2.0
+          kafka_version: 3.2.3
+        - scala: 2.13
+          kafka_version: 3.3.2
+        - scala: 2.13
+          kafka_version: 3.4.0
 
     steps:
     - name: Checkout

From ede282bb7274783ff1da8f5cdb1ca333370e9a48 Mon Sep 17 00:00:00 2001
From: Stephen Sorriaux <stephen.sorriaux@gmail.com>
Date: Fri, 10 Mar 2023 18:03:34 -0500
Subject: [PATCH 33/34] Add testing to GA

---
 .github/workflows/package.yml | 69 ++++++++++++++++++++++++++++++++++-
 test/docker-compose.yml       |  4 +-
 test/verifyImageLabels.sh     | 13 ++++---
 3 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 45743ab0..c2474cfe 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -21,32 +21,50 @@ jobs:
         include:
         - scala: 2.12
           kafka_version: 2.1.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.2.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.3.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.4.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.12
           kafka_version: 2.5.1
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
           kafka_version: 2.6.3
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
           kafka_version: 2.7.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
           kafka_version: 2.8.2
+          extra_compose_file: "test/docker-compose-kafka.yml"
         - scala: 2.13
           kafka_version: 3.0.2
+          extra_compose_file: "test/docker-compose-kafka3.yml"
         - scala: 2.13
           kafka_version: 3.1.2
+          extra_compose_file: "test/docker-compose-kafka3.yml"
         - scala: 2.13
           kafka_version: 3.2.3
+          extra_compose_file: "test/docker-compose-kafka3.yml"
         - scala: 2.13
           kafka_version: 3.3.2
+          extra_compose_file: "test/docker-compose-kafka3.yml"
         - scala: 2.13
           kafka_version: 3.4.0
+          extra_compose_file: "test/docker-compose-kafka3.yml"
 
     steps:
+    - name: Install needed libs
+      run: |
+        sudo apt update
+        sudo apt install -y shellcheck
+
     - name: Checkout
       uses: actions/checkout@v3
 
@@ -65,9 +83,56 @@ jobs:
       id: setup
       run: |
         echo BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGE_VERSION=${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_ENV
         echo DOCKER_IMAGE_TAG=${{ env.IMAGE_DEST }}:${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_OUTPUT
 
-    - name: Build and push
+    - name: Build image
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        platforms: linux/amd64,linux/arm64
+        push: false
+        tags: ${{ steps.setup.outputs.DOCKER_IMAGE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          kafka_version=${{ matrix.kafka_version }}
+          scala_version=${{ matrix.scala }}
+          vcs_ref=${{ github.sha }}
+          build_date=${{ steps.setup.outputs.BUILD_DATE }}
+
+    - name: Create test env using docker-compose
+      run: |
+        docker-compose -f test/docker-compose.yml -f ${{ matrix.extra_compose_file }} up -d zookeeper kafka_1 kafka_2
+
+    - name: Perform shellcheck
+      run: |
+        echo "Shellcheck main source files"
+        shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
+        cd test
+        echo "Shellcheck the tests"
+        shellcheck -x -e SC1090 -s bash *.sh **/*.sh
+
+    - name: Perform tests for image
+      run: |
+        cd test
+        echo "Verify docker image's label"
+        ./verifyImageLabels.sh
+        echo "Wait for containers to start"
+        sleep 5
+        # docker-compose logs
+        docker ps -a
+        echo "Run tests"
+        ./runAllTests.sh
+        echo "End-to-End scenario tests"
+        cd scenarios
+        ./runJmxScenario.sh
+
+    - name: Destroy test env using docker-compose
+      run: |
+        docker-compose down
+
+    - name: Push image
       uses: docker/build-push-action@v4
       with:
         context: .
@@ -80,4 +145,4 @@ jobs:
           kafka_version=${{ matrix.kafka_version }}
           scala_version=${{ matrix.scala }}
           vcs_ref=${{ github.sha }}
-          build_date=${{ steps.setup.outputs.BUILD_DATE }}
\ No newline at end of file
+          build_date=${{ steps.setup.outputs.BUILD_DATE }}
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
index 78d7f35d..ad5dd42f 100644
--- a/test/docker-compose.yml
+++ b/test/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.1'
 
 x-kafka-defaults: &kafka-defaults
-  image: wurstmeister/kafka
+  image: "ssorriaux/kafka:${DOCKER_IMAGE_VERSION}"
   ports:
     - "9092"
   volumes:
@@ -24,7 +24,7 @@ services:
       KAFKA_BROKER_ID: 2
 
   kafkatest:
-    image: wurstmeister/kafka
+    image: "ssorriaux/kafka:${DOCKER_IMAGE_VERSION}"
     environment:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:
diff --git a/test/verifyImageLabels.sh b/test/verifyImageLabels.sh
index f59c9985..7821d274 100755
--- a/test/verifyImageLabels.sh
+++ b/test/verifyImageLabels.sh
@@ -1,17 +1,18 @@
 #!/bin/bash -e
 
-VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' wurstmeister/kafka)
+VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' ssorriaux/kafka:"${DOCKER_IMAGE_VERSION}")
 echo "VCS_REF=$VCS_REF"
 if [ -z "$VCS_REF" ] || [ "$VCS_REF" = "unspecified" ]; then
   echo "org.label-schema.vcs-ref is empty or unspecified"
   exit 1
 fi
-if ! git cat-file -e "$VCS_REF^{commit}"; then
-  echo "$VCS_REF Not a valid git commit"
-  exit 1
-fi
+#
+# if ! git cat-file -e "$VCS_REF^{commit}"; then
+#   echo "$VCS_REF Not a valid git commit"
+#   exit 1
+# fi
 
-BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' wurstmeister/kafka)
+BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' ssorriaux/kafka:"${DOCKER_IMAGE_VERSION}")
 echo "BUILD_DATE=$BUILD_DATE"
 if ! date -d "$BUILD_DATE"; then
   echo "$BUILD_DATE Not a valid date"

From 7f47e7942288f80102fc2c3105825063e2b8e09a Mon Sep 17 00:00:00 2001
From: Stephen Sorriaux <stephen.sorriaux@gmail.com>
Date: Fri, 10 Mar 2023 19:22:46 -0500
Subject: [PATCH 34/34] Update README.md

---
 README.md | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 723b584d..0d9709ee 100644
--- a/README.md
+++ b/README.md
@@ -1,16 +1,18 @@
-[![Docker Pulls](https://img.shields.io/docker/pulls/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/)
-[![Docker Stars](https://img.shields.io/docker/stars/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/)
-[![](https://images.microbadger.com/badges/version/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own version badge on microbadger.com")
-[![](https://images.microbadger.com/badges/image/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own image badge on microbadger.com")
-[![Build Status](https://app.travis-ci.com/wurstmeister/kafka-docker.svg?branch=master)](https://app.travis-ci.com/wurstmeister/kafka-docker)
+[![Docker Pulls](https://img.shields.io/docker/pulls/ssorriaux/kafka.svg)](https://hub.docker.com/r/ssorriaux/kafka/)
+[![Docker Stars](https://img.shields.io/docker/stars/ssorriaux/kafka.svg)](https://hub.docker.com/r/ssorriaux/kafka/)
+[![](https://images.microbadger.com/badges/version/ssorriaux/kafka.svg)](https://microbadger.com/images/ssorriaux/kafka "Get your own version badge on microbadger.com")
+[![](https://images.microbadger.com/badges/image/ssorriaux/kafka.svg)](https://microbadger.com/images/ssorriaux/kafka "Get your own image badge on microbadger.com")
+[![Build Status](https://app.travis-ci.com/ssorriaux/kafka-docker.svg?branch=master)](https://app.travis-ci.com/ssorriaux/kafka-docker)
 
 
 kafka-docker
 ============
 
+**Fork of [wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) since no update was provided since May, 2022.**
+
 Dockerfile for [Apache Kafka](http://kafka.apache.org/)
 
-The image is available directly from [Docker Hub](https://hub.docker.com/r/wurstmeister/kafka/)
+The image is available directly from [Docker Hub](https://hub.docker.com/r/ssorriaux/kafka/)
 
 Tags and releases
 -----------------
@@ -18,7 +20,7 @@ Tags and releases
 All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `<scala version>-<kafka version>`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). To list all available tags:
 
 ```
-curl -s https://registry.hub.docker.com/v2/repositories/wurstmeister/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*'
+curl -s https://registry.hub.docker.com/v2/repositories/ssorriaux/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*'
 ```
 
 Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image.
@@ -34,7 +36,7 @@ Everytime the image is updated, all tags will be pushed with the latest updates.
 ## Pre-Requisites
 
 - install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/)
-- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/wurstmeister/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.)
+- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/ssorriaux/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.)
 - if you want to customize any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'```
 - Kafka's log4j usage can be customized by adding environment variables prefixed with ```LOG4J_```. These will be mapped to ```log4j.properties```. For example: ```LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER=DEBUG, authorizerAppender```
 
@@ -56,7 +58,7 @@ Destroy a cluster:
 
 ## Note
 
-The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/wurstmeister/kafka-docker/blob/master/docker-compose-single-broker.yml):
+The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/StephenSorriaux/kafka-docker/blob/master/docker-compose-single-broker.yml):
 
 - ```docker-compose -f docker-compose-single-broker.yml up```