diff --git a/.travis.yml b/.travis.yml index 04fe8439..3253c94f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,27 +8,56 @@ services: # This version will be also tagged as 'latest' env: global: - - LATEST="2.13-2.8.1" + - LATEST="2.13-3.2.0" # Build recommended versions based on: http://kafka.apache.org/downloads matrix: include: - scala: 2.12 - env: KAFKA_VERSION=2.1.1 + env: + - KAFKA_VERSION=2.1.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.2.2 + env: + - KAFKA_VERSION=2.2.2 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.3.1 + env: + - KAFKA_VERSION=2.3.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.4.1 + env: + - KAFKA_VERSION=2.4.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.5.1 + env: + - KAFKA_VERSION=2.5.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.13 - env: KAFKA_VERSION=2.6.3 + env: + - KAFKA_VERSION=2.6.3 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.13 - env: KAFKA_VERSION=2.7.2 + env: + - KAFKA_VERSION=2.7.2 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.13 - env: KAFKA_VERSION=2.8.1 + env: + - KAFKA_VERSION=2.8.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml + - scala: 2.13 + env: + - KAFKA_VERSION=3.0.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml + - scala: 2.13 + env: + - KAFKA_VERSION=3.1.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml + - scala: 2.13 + env: + - KAFKA_VERSION=3.2.0 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml + # Upgrade Docker Engine so we can use buildx before_install: @@ -60,7 +89,7 @@ install: - docker pull confluentinc/cp-kafkacat before_script: - - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2 + - docker-compose -f test/docker-compose.yml -f ${EXTRA_COMPOSE_FILE} up -d zookeeper kafka_1 kafka_2 script: # Shellcheck main source files diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ed2cf9..fa848a48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ Changelog ========= Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date. +20-May-2022 +----------- +- Add support for Kafka `3.0.1`,`3.1.1` and `3.2.0` 09-Apr-2022 ---------- diff --git a/Dockerfile b/Dockerfile index 6983799c..24c0cc40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM openjdk:11-jre-slim +FROM eclipse-temurin:17-jre -ARG kafka_version=2.8.1 +ARG kafka_version=3.4.0 ARG scala_version=2.13 ARG vcs_ref=unspecified ARG build_date=unspecified @@ -28,9 +28,9 @@ RUN set -eux ; \ apt-get install -y --no-install-recommends jq net-tools curl wget ; \ ### BEGIN docker for CI tests apt-get install -y --no-install-recommends gnupg lsb-release ; \ - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg ; \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg ; \ echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null ; \ apt-get update ; \ apt-get install -y --no-install-recommends docker-ce-cli ; \ diff --git a/README.md b/README.md index ca0ce84a..723b584d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to sp ## Advertised hostname -You can configure the advertised hostname in different ways +You can configure the advertised hostname in different ways (Note that as of kafka 3 KAFKA_ADVERTISED_HOST_NAME,KAFKA_ADVERTISED_PORT KAFKA_PORT and KAFKA_HOST_NAME is no longer supported) 1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME``` 2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"``` diff --git a/broker-list.sh b/broker-list.sh index 5c5ee2d7..4648330d 100755 --- a/broker-list.sh +++ b/broker-list.sh @@ -1,5 +1,6 @@ #!/bin/bash CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') -BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) +# We are just interested in ipv4 addresses, so ignore ipv6 addresses +BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | grep '0.0.0.0' | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) echo "${BROKERS//$'\n'/,}" diff --git a/create-topics.sh b/create-topics.sh index 0bacf7b5..21bc83fb 100755 --- a/create-topics.sh +++ b/create-topics.sh @@ -11,15 +11,34 @@ fi start_timeout_exceeded=false count=0 step=10 -while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do - echo "waiting for kafka to be ready" - sleep $step; - count=$((count + step)) - if [ $count -gt $START_TIMEOUT ]; then - start_timeout_exceeded=true - break + +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" +# since 3.0.0 there is no --zookeeper option anymore, so we have to use the +# --bootstrap-server option. +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + if [[ -v KAFKA_LISTENERS ]]; then + PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' ) + CONNECT_OPTS="--bootstrap-server localhost:${PORT}" + else + CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}" fi -done +else + PORT="$KAFKA_PORT" + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" +fi + +if [[ ! -v PORT ]]; then + while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do + echo "waiting for kafka to be ready" + sleep $step; + count=$((count + step)) + if [ $count -gt $START_TIMEOUT ]; then + start_timeout_exceeded=true + break + fi + done +fi if $start_timeout_exceeded; then echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" @@ -27,8 +46,6 @@ if $start_timeout_exceeded; then fi # introduced in 0.10. In earlier versions, this will fail because the topic already exists. -# shellcheck disable=SC1091 -source "/usr/bin/versions.sh" if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then KAFKA_0_10_OPTS="--if-not-exists" fi @@ -45,7 +62,7 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ --create \\ - --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ + ${CONNECT_OPTS} \\ --topic ${topicConfig[0]} \\ --partitions ${topicConfig[1]} \\ --replication-factor ${topicConfig[2]} \\ diff --git a/docker-compose-single-broker.yml b/docker-compose-single-broker.yml index 4d8e9f51..e6f45373 100644 --- a/docker-compose-single-broker.yml +++ b/docker-compose-single-broker.yml @@ -9,7 +9,8 @@ services: ports: - "9092:9092" environment: - KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092 + KAFKA_LISTENERS: PLAINTEXT://:9092 KAFKA_CREATE_TOPICS: "test:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: diff --git a/docker-compose.yml b/docker-compose.yml index b4f77b38..73e0d591 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,8 @@ services: - "9092" environment: DOCKER_API_VERSION: 1.22 - KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092 + KAFKA_LISTENERS: PLAINTEXT://:9092 volumes: - /var/run/docker.sock:/var/run/docker.sock restart: unless-stopped diff --git a/start-kafka.sh b/start-kafka.sh index b48280a4..148fa577 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -1,5 +1,8 @@ #!/bin/bash -e +# shellcheck disable=SC1091 +source /usr/bin/versions.sh + # Allow specific kafka versions to perform any unique bootstrap operations OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh" if [[ -x "$OVERRIDE_FILE" ]]; then @@ -10,24 +13,45 @@ fi # Store original IFS config, so we can restore it at various stages ORIG_IFS=$IFS -if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then - echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" - exit 1 +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + if [[ -v KAFKA_ADVERTISED_HOST_NAME ]]; then + echo "ERROR: KAFKA_ADVERTISED_HOST_NAME is removed as of kafka 3, remove KAFKA_ADVERTISED_HOST_NAME=$KAFKA_ADVERTISED_HOST_NAME from your config" + exit 1 + fi + if [[ -v KAFKA_ADVERTISED_PORT ]]; then + echo "ERROR: KAFKA_ADVERTISED_PORT is removed as of kafka 3, remove KAFKA_ADVERTISED_PORT=$KAFKA_ADVERTISED_PORT from your config" + exit 1 + fi + if [[ -v KAFKA_PORT ]]; then + echo "ERROR: KAFKA_PORT is removed as of kafka 3, remove KAFKA_PORT=$KAFKA_PORT from your config" + exit 1 + fi + if [[ -v KAFKA_HOST_NAME ]]; then + echo "ERROR: KAFKA_HOST_NAME is removed as of kafka 3, remove KAFKA_HOST_NAME=$KAFKA_HOST_NAME from your config" + exit 1 + fi +else + if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then + echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" + exit 1 + fi + if [[ -z "$KAFKA_PORT" ]]; then + export KAFKA_PORT=9092 + fi fi -if [[ -z "$KAFKA_PORT" ]]; then - export KAFKA_PORT=9092 -fi create-topics.sh & unset KAFKA_CREATE_TOPICS -if [[ -z "$KAFKA_ADVERTISED_PORT" && \ - -z "$KAFKA_LISTENERS" && \ - -z "$KAFKA_ADVERTISED_LISTENERS" && \ - -S /var/run/docker.sock ]]; then - KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) - export KAFKA_ADVERTISED_PORT +if [[ "$MAJOR_VERSION" -lt "3" ]]; then + if [[ -z "$KAFKA_ADVERTISED_PORT" && \ + -z "$KAFKA_LISTENERS" && \ + -z "$KAFKA_ADVERTISED_LISTENERS" && \ + -S /var/run/docker.sock ]]; then + KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) + export KAFKA_ADVERTISED_PORT + fi fi if [[ -z "$KAFKA_BROKER_ID" ]]; then @@ -81,19 +105,30 @@ if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then fi # Try and configure minimal settings or exit with error if there isn't enough information -if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then - if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then - echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS" - exit 1 - elif [[ -z "$HOSTNAME_VALUE" ]]; then - echo "ERROR: No listener or advertised hostname configuration provided in environment." - echo " Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME" - exit 1 - fi +if [[ "$MAJOR_VERSION" -lt "3" ]]; then + if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then + if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then + echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS" + exit 1 + elif [[ -z "$HOSTNAME_VALUE" ]]; then + echo "ERROR: No listener or advertised hostname configuration provided in environment." + echo " Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME" + exit 1 + fi - # Maintain existing behaviour - # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined. - export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" + # Maintain existing behaviour + # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined. + export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" + fi +else + if [[ "$HOSTNAME_VALUE" ]]; then + if [[ -z "$KAFKA_ADVERTISED_LISTENERS" ]]; then + export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$HOSTNAME_VALUE:9092" + if [[ -z "$KAFKA_LISTENERS" ]]; then + export KAFKA_LISTENERS=PLAINTEXT://:9092 + fi + fi + fi fi #Issue newline to config file in case there is not one already diff --git a/test/0.0/test.create-topics-custom-separator.kafka.sh b/test/0.0/test.create-topics-custom-separator.kafka.sh index abd031c1..c26e8131 100755 --- a/test/0.0/test.create-topics-custom-separator.kafka.sh +++ b/test/0.0/test.create-topics-custom-separator.kafka.sh @@ -20,13 +20,25 @@ testCreateTopicsCustomSeparator() { create-topics.sh + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no --zookeeper option anymore, so we have to use the + # --bootstrap-server option with a random broker + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" + else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" + fi + # Loop through each array, validate that topic exists for i in "${!TOPICS[@]}"; do TOPIC=${TOPICS[i]} echo "Validating topic '$TOPIC'" - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") + # shellcheck disable=SC2086 + EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC") if [[ "$EXISTS" != "$TOPIC" ]]; then echo "$TOPIC topic not created" return 1 @@ -36,8 +48,18 @@ testCreateTopicsCustomSeparator() { return 0 } +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" + +# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}') +else + PORT="$KAFKA_PORT" +fi + # mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } +function netstat() { echo "1 2 3 :$PORT"; } export -f netstat testCreateTopicsCustomSeparator diff --git a/test/0.0/test.start-kafka-advertised-host.kafka.sh b/test/0.0/test.start-kafka-advertised-host.kafka.sh index 1791d60b..427df650 100755 --- a/test/0.0/test.start-kafka-advertised-host.kafka.sh +++ b/test/0.0/test.start-kafka-advertised-host.kafka.sh @@ -16,5 +16,14 @@ testAdvertisedHost() { assertAbsent 'advertised.listeners' assertAbsent 'listeners' } +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" -testAdvertisedHost +# since 3.0.0 there is no --zookeeper option anymore, so we have to use the +# --bootstrap-server option with a random broker +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + echo "with kafka from version 3.0.0 'advertised.port' and 'advertised.host.name' are removed, making this test obosolete" + echo "See: https://github.com/apache/kafka/pull/10872" +else + testAdvertisedHost +fi \ No newline at end of file diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh index 5f0167c3..59703095 100755 --- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh +++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh @@ -3,9 +3,16 @@ source test.functions testKafkaEnv() { + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf" + # since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + + export KAFKA_ADVERTISED_HOST_NAME="testhost" + fi # When the script is invoked source "$START_KAFKA" @@ -22,4 +29,4 @@ testKafkaEnv() { echo " > Set KAFKA_OPTS=$KAFKA_OPTS" } -testKafkaEnv +testKafkaEnv \ No newline at end of file diff --git a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh index df7b4018..720c2f39 100755 --- a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh +++ b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh @@ -3,8 +3,15 @@ source test.functions testKafkaOpts() { + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + export KAFKA_ADVERTISED_HOST_NAME="testhost" + fi # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" + # .. and a CUSTOM_INIT_SCRIPT with spaces export CUSTOM_INIT_SCRIPT="export KAFKA_OPTS=-Djava.security.auth.login.config=/kafka_server_jaas.conf" @@ -20,4 +27,4 @@ testKafkaOpts() { echo " > Set KAFKA_OPTS=$KAFKA_OPTS" } -testKafkaOpts +testKafkaOpts \ No newline at end of file diff --git a/test/0.0/test.start-kafka-host-name.kafka.sh b/test/0.0/test.start-kafka-host-name.kafka.sh index 08dc2113..5aabf205 100755 --- a/test/0.0/test.start-kafka-host-name.kafka.sh +++ b/test/0.0/test.start-kafka-host-name.kafka.sh @@ -8,11 +8,20 @@ testHostnameCommand() { # When the script is invoked source "$START_KAFKA" + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" - # Then the configuration uses the value from the command - assertExpectedConfig 'advertised.host.name=my-host' - assertAbsent 'advertised.listeners' - assertAbsent 'listeners' + # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + # Then the configuration uses the value from the command + assertExpectedConfig 'advertised.listeners=PLAINTEXT://my-host:9092' + assertExpectedConfig 'listeners=PLAINTEXT://:9092' + else + # Then the configuration uses the value from the command + assertExpectedConfig 'advertised.host.name=my-host' + assertAbsent 'advertised.listeners' + assertAbsent 'listeners' + fi } testHostnameCommand diff --git a/test/0.0/test.start-kafka-log4j-config.kafka.sh b/test/0.0/test.start-kafka-log4j-config.kafka.sh index da4ff28e..f1d68614 100755 --- a/test/0.0/test.start-kafka-log4j-config.kafka.sh +++ b/test/0.0/test.start-kafka-log4j-config.kafka.sh @@ -4,7 +4,14 @@ source test.functions testLog4jConfig() { # Given Log4j overrides are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + export KAFKA_ADVERTISED_HOST_NAME="testhost" + fi + export LOG4J_LOGGER_KAFKA=DEBUG # When the script is invoked @@ -13,5 +20,4 @@ testLog4jConfig() { # Then the configuration file is correct assertExpectedLog4jConfig "log4j.logger.kafka=DEBUG" } - -testLog4jConfig +testLog4jConfig \ No newline at end of file diff --git a/test/0.0/test.start-kafka-restart.kafka.sh b/test/0.0/test.start-kafka-restart.kafka.sh index 68a25598..6d526c4b 100755 --- a/test/0.0/test.start-kafka-restart.kafka.sh +++ b/test/0.0/test.start-kafka-restart.kafka.sh @@ -3,16 +3,27 @@ source test.functions testRestart() { - # Given a hostname is provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + # since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + # Given a hostname is provided + export KAFKA_ADVERTISED_HOST_NAME="testhost" + else + export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://testhost:9092" + fi # When the container is restarted (Script invoked multiple times) source "$START_KAFKA" source "$START_KAFKA" - # Then the configuration file only has one instance of the config - assertExpectedConfig 'advertised.host.name=testhost' - assertAbsent 'listeners' + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + # Then the configuration file only has one instance of the config + assertExpectedConfig 'advertised.host.name=testhost' + assertAbsent 'listeners' + else + assertExpectedConfig 'advertised.listeners=PLAINTEXT://testhost:9092' + assertAbsent 'listeners' + fi } - -testRestart +testRestart \ No newline at end of file diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh index b318512f..c8a1a6a1 100755 --- a/test/0.10/test.create-topics.kafka.sh +++ b/test/0.10/test.create-topics.kafka.sh @@ -6,29 +6,49 @@ testCreateTopics() { NOW=$(date +%s) # TOPICS array contains the topic name to create / validate - # CLEANUP array contains the expected cleanup policy configuration for the topic + # CONFIG array contains the expected cleanup policy configuration for the topic TOPICS[0]="default-$NOW" - CLEANUP[0]="" + CONFIG[0]="" TOPICS[1]="compact-$NOW" - CLEANUP[1]="compression.type=snappy,cleanup.policy=compact" + CONFIG[1]=$(echo -e "compression.type=snappy\ncleanup.policy=compact" | sort \ + | tr '\n' ',' \ + | sed 's/,$//') KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh - # Loop through each array, validate that topic exists, and correct cleanup policy is set + + # since 3.0.0 there is no --zookeeper option anymore, so we have to use the + # --bootstrap-server option with a random broker + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" + else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" + fi + + # Loop through each array, validate that topic exists, and correct configuration is set for i in "${!TOPICS[@]}"; do TOPIC=${TOPICS[i]} echo "Validating topic '$TOPIC'" - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - POLICY=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --describe --topic "$TOPIC" | awk -F'Configs:' '{print $2}' | xargs) + # shellcheck disable=SC2086 + EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC") + # shellcheck disable=SC2086 + ACTUAL_CONFIG=$(/opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe \ + | cut -d'{' -f1 \ + | grep -oE '(compression.type|cleanup.policy)=([^ ,]+)' \ + | sort \ + | tr '\n' ',' \ + | sed 's/,$//') - RESULT="$EXISTS:$POLICY" - EXPECTED="$TOPIC:${CLEANUP[i]}" + RESULT="$EXISTS:$ACTUAL_CONFIG" + EXPECTED="$TOPIC:${CONFIG[i]}" if [[ "$RESULT" != "$EXPECTED" ]]; then - echo "$TOPIC topic not configured correctly: '$RESULT'" + echo "$TOPIC topic not configured correctly:" + echo " Actual: '$RESULT'" + echo " Expected: '$EXPECTED'" return 1 fi done @@ -36,8 +56,18 @@ testCreateTopics() { return 0 } +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" + +# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}') +else + PORT="$KAFKA_PORT" +fi + # mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } +function netstat() { echo "1 2 3 :$PORT"; } export -f netstat testCreateTopics diff --git a/test/Readme.md b/test/Readme.md index cdc72c19..ba50b714 100644 --- a/test/Readme.md +++ b/test/Readme.md @@ -8,7 +8,10 @@ To execute ``` cd test -docker-compose up -d zookeeper kafka_1 kafka_2 +# if kafka veresion < 3 +docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka_1 kafka_2 +# if kafka veresion >= 3 +docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka_1 kafka_2 ./runAllTests.sh ``` @@ -18,7 +21,7 @@ Run selected tests ### Kafka ``` -docker-compose run --rm kafkatest +docker-compose run -e BROKER_LIST=$(./internal-broker-list.sh) --rm kafkatest ``` ### Kafkacat diff --git a/test/docker-compose-kafka.yml b/test/docker-compose-kafka.yml new file mode 100644 index 00000000..6786c860 --- /dev/null +++ b/test/docker-compose-kafka.yml @@ -0,0 +1,23 @@ +version: '2.1' + +x-kafka-environment-defaults: &kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + +services: + kafka_1: + environment: + <<: *kafka-environment-defaults + + kafka_2: + environment: + <<: *kafka-environment-defaults + + kafkatest: + environment: + <<: *kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 \ No newline at end of file diff --git a/test/docker-compose-kafka3.yml b/test/docker-compose-kafka3.yml new file mode 100644 index 00000000..2fa0a743 --- /dev/null +++ b/test/docker-compose-kafka3.yml @@ -0,0 +1,24 @@ +version: '2.1' + +x-kafka-environment-defaults: &kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + +services: + kafka_1: + environment: + <<: *kafka-environment-defaults + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_1:9092 + + kafka_2: + environment: + <<: *kafka-environment-defaults + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_2:9092 + + kafkatest: + environment: + <<: *kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 \ No newline at end of file diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 42316de2..78d7f35d 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -7,12 +7,6 @@ x-kafka-defaults: &kafka-defaults volumes: - /var/run/docker.sock:/var/run/docker.sock -x-kafka-environment-defaults: &kafka-environment-defaults - HOSTNAME_COMMAND: "echo $$(hostname)" - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - services: zookeeper: image: wurstmeister/zookeeper @@ -22,19 +16,16 @@ services: <<: *kafka-defaults container_name: test_kafka_1 environment: - <<: *kafka-environment-defaults KAFKA_BROKER_ID: 1 kafka_2: <<: *kafka-defaults container_name: test_kafka_2 environment: - <<: *kafka-environment-defaults KAFKA_BROKER_ID: 2 kafkatest: image: wurstmeister/kafka environment: - KAFKA_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - .:/tests @@ -49,7 +40,7 @@ services: image: confluentinc/cp-kafkacat:5.0.0 environment: - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.8.1} + - KAFKA_VERSION=${KAFKA_VERSION-3.2.0} volumes: - .:/tests working_dir: /tests diff --git a/test/runAllTests.sh b/test/runAllTests.sh index a575b21f..da81e67d 100755 --- a/test/runAllTests.sh +++ b/test/runAllTests.sh @@ -7,12 +7,12 @@ echo "BROKER_LIST=$BROKER_LIST" runAll() { # Tests that require kafka - docker-compose run --rm kafkatest + docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkatest RESULT=$? if [[ $RESULT -eq 0 ]]; then # Tests that require kafkacat - docker-compose run --rm kafkacattest + docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkacattest RESULT=$? fi diff --git a/test/scenarios/jmx/docker-compose.yml b/test/scenarios/jmx/docker-compose.yml index aba78bf2..b86f4ddf 100644 --- a/test/scenarios/jmx/docker-compose.yml +++ b/test/scenarios/jmx/docker-compose.yml @@ -10,11 +10,10 @@ services: - "9092" - "1099" environment: - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099" JMX_PORT: 1099 volumes: @@ -31,7 +30,6 @@ services: test: image: wurstmeister/kafka environment: - KAFKA_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - .:/scenario