From 5b918596ceb8dcc1d8b15cccaf4e204da32dded9 Mon Sep 17 00:00:00 2001 From: Hermann Mayer Date: Mon, 11 Oct 2021 12:17:50 +0200 Subject: [PATCH 1/3] Add support for Kafka 3.0.0. Signed-off-by: Hermann Mayer --- .travis.yml | 4 ++- CHANGELOG.md | 5 +++ Dockerfile | 2 +- README.md | 1 + broker-list.sh | 3 +- create-topics.sh | 10 +++++- ...st.create-topics-custom-separator.kafka.sh | 14 +++++++- test/0.10/test.create-topics.kafka.sh | 36 ++++++++++++++----- test/docker-compose.yml | 2 +- test/runAllTests.sh | 4 +-- 10 files changed, 65 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b32acb9..3f67746c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ services: # This version will be also tagged as 'latest' env: global: - - LATEST="2.13-2.7.1" + - LATEST="2.13-3.0.0" # Build recommended versions based on: http://kafka.apache.org/downloads matrix: @@ -29,6 +29,8 @@ matrix: env: KAFKA_VERSION=2.6.0 - scala: 2.13 env: KAFKA_VERSION=2.7.1 + - scala: 2.13 + env: KAFKA_VERSION=3.0.0 install: - docker --version diff --git a/CHANGELOG.md b/CHANGELOG.md index ebefdf17..0fad1d59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ Changelog Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date. +11-Oct-2021 +----------- + +- Add support for Kafka `3.0.0` + 19-July-2021 ---------- diff --git a/Dockerfile b/Dockerfile index 668ac714..753ec337 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM azul/zulu-openjdk-alpine:8u292-8.54.0.21 -ARG kafka_version=2.7.1 +ARG kafka_version=3.0.0 ARG scala_version=2.13 ARG glibc_version=2.31-r0 ARG vcs_ref=unspecified diff --git a/README.md b/README.md index 629ddbd9..acc78d90 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Tags and releases All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `-`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). Available tags are: +- `2.13-3.0.0` - `2.13-2.7.1` - `2.13-2.6.0` - `2.12-2.5.0` diff --git a/broker-list.sh b/broker-list.sh index 5c5ee2d7..4648330d 100755 --- a/broker-list.sh +++ b/broker-list.sh @@ -1,5 +1,6 @@ #!/bin/bash CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') -BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) +# We are just interested in ipv4 addresses, so ignore ipv6 addresses +BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | grep '0.0.0.0' | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) echo "${BROKERS//$'\n'/,}" diff --git a/create-topics.sh b/create-topics.sh index 0bacf7b5..3ccbe194 100755 --- a/create-topics.sh +++ b/create-topics.sh @@ -33,6 +33,14 @@ if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSI KAFKA_0_10_OPTS="--if-not-exists" fi +# since 3.0.0 there is no --zookeeper option anymore, so we have to use the +# --bootstrap-server option with a random broker +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" +else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" +fi + # Expected format: # name:partitions:replicas:cleanup.policy IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do @@ -45,7 +53,7 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ --create \\ - --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ + ${CONNECT_OPTS} \\ --topic ${topicConfig[0]} \\ --partitions ${topicConfig[1]} \\ --replication-factor ${topicConfig[2]} \\ diff --git a/test/0.0/test.create-topics-custom-separator.kafka.sh b/test/0.0/test.create-topics-custom-separator.kafka.sh index abd031c1..4ad32bd3 100755 --- a/test/0.0/test.create-topics-custom-separator.kafka.sh +++ b/test/0.0/test.create-topics-custom-separator.kafka.sh @@ -20,13 +20,25 @@ testCreateTopicsCustomSeparator() { create-topics.sh + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no --zookeeper option anymore, so we have to use the + # --bootstrap-server option with a random broker + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" + else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" + fi + # Loop through each array, validate that topic exists for i in "${!TOPICS[@]}"; do TOPIC=${TOPICS[i]} echo "Validating topic '$TOPIC'" - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") + # shellcheck disable=SC2086 + EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC") if [[ "$EXISTS" != "$TOPIC" ]]; then echo "$TOPIC topic not created" return 1 diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh index e9dda7bd..d0088291 100755 --- a/test/0.10/test.create-topics.kafka.sh +++ b/test/0.10/test.create-topics.kafka.sh @@ -8,27 +8,47 @@ testCreateTopics() { # TOPICS array contains the topic name to create / validate # CLEANUP array contains the expected cleanup policy configuration for the topic TOPICS[0]="default-$NOW" - CLEANUP[0]="" + CONFIG[0]="" TOPICS[1]="compact-$NOW" - CLEANUP[1]="compact,compression.type=snappy" + CONFIG[1]="cleanup.policy=compact,compression.type=snappy" KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh - # Loop through each array, validate that topic exists, and correct cleanup policy is set + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no --zookeeper option anymore, so we have to use the + # --bootstrap-server option with a random broker + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" + else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" + fi + + # Loop through each array, validate that topic exists, and correct configuration is set for i in "${!TOPICS[@]}"; do TOPIC=${TOPICS[i]} echo "Validating topic '$TOPIC'" - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - POLICY=$(/opt/kafka/bin/kafka-configs.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --entity-type topics --entity-name "$TOPIC" --describe | grep 'Configs for topic' | awk -F'cleanup.policy=' '{print $2}') + # shellcheck disable=SC2086 + EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC") + # shellcheck disable=SC2086 + ACTUAL_CONFIG=$(/opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe \ + | cut -d'{' -f1 \ + | grep -oE '(compression.type|cleanup.policy)=([^ ,]+)' \ + | sort \ + | tr '\n' ',' \ + | sed 's/,$//') - RESULT="$EXISTS:$POLICY" - EXPECTED="$TOPIC:${CLEANUP[i]}" + RESULT="$EXISTS:$ACTUAL_CONFIG" + EXPECTED="$TOPIC:${CONFIG[i]}" if [[ "$RESULT" != "$EXPECTED" ]]; then - echo "$TOPIC topic not configured correctly: '$RESULT'" + echo "$TOPIC topic not configured correctly:" + echo " Actual: '$RESULT'" + echo " Expected: '$EXPECTED'" return 1 fi done diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 0aa8ad9b..05a7a453 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -49,7 +49,7 @@ services: image: confluentinc/cp-kafkacat:5.0.0 environment: - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.7.1} + - KAFKA_VERSION=${KAFKA_VERSION-3.0.0} volumes: - .:/tests working_dir: /tests diff --git a/test/runAllTests.sh b/test/runAllTests.sh index a575b21f..da81e67d 100755 --- a/test/runAllTests.sh +++ b/test/runAllTests.sh @@ -7,12 +7,12 @@ echo "BROKER_LIST=$BROKER_LIST" runAll() { # Tests that require kafka - docker-compose run --rm kafkatest + docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkatest RESULT=$? if [[ $RESULT -eq 0 ]]; then # Tests that require kafkacat - docker-compose run --rm kafkacattest + docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkacattest RESULT=$? fi From b988d2ca699fb8a9f821d0221dae240935b16569 Mon Sep 17 00:00:00 2001 From: Hermann Mayer Date: Tue, 12 Oct 2021 12:58:22 +0200 Subject: [PATCH 2/3] one of many Signed-off-by: Hermann Mayer --- start-kafka.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/start-kafka.sh b/start-kafka.sh index b48280a4..a016c285 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +source /usr/bin/versions.sh + # Allow specific kafka versions to perform any unique bootstrap operations OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh" if [[ -x "$OVERRIDE_FILE" ]]; then @@ -26,7 +28,7 @@ if [[ -z "$KAFKA_ADVERTISED_PORT" && \ -z "$KAFKA_LISTENERS" && \ -z "$KAFKA_ADVERTISED_LISTENERS" && \ -S /var/run/docker.sock ]]; then - KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) + KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) export KAFKA_ADVERTISED_PORT fi @@ -96,6 +98,16 @@ if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" fi +# `advertised.port` and `advertised.host.name` are removed with Kafka 3.0.0 +# See: https://github.com/apache/kafka/pull/10872 +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + if [ -z "$KAFKA_ADVERTISED_LISTENERS" ]; then + if [ -n "${KAFKA_ADVERTISED_HOST_NAME}" ]; then + export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME}:${KAFKA_PORT}" + fi + fi +fi + #Issue newline to config file in case there is not one already echo "" >> "$KAFKA_HOME/config/server.properties" From 8e29df3ea0c92c695bfd3a0642ac3f9607d98fd1 Mon Sep 17 00:00:00 2001 From: Hermann Mayer Date: Tue, 12 Oct 2021 14:08:42 +0200 Subject: [PATCH 3/3] one of many Signed-off-by: Hermann Mayer --- start-kafka.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/start-kafka.sh b/start-kafka.sh index a016c285..fb56deb5 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -1,5 +1,6 @@ #!/bin/bash -e +# shellcheck disable=SC1091 source /usr/bin/versions.sh # Allow specific kafka versions to perform any unique bootstrap operations