diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index a3cfbb83..c2474cfe 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -8,7 +8,7 @@ on: branches: ["master"] env: - LATEST: "2.13-2.8.1" + LATEST: "2.13-3.2.0" IMAGE_DEST: "ssorriaux/kafka" jobs: @@ -21,22 +21,50 @@ jobs: include: - scala: 2.12 kafka_version: 2.1.1 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.12 kafka_version: 2.2.2 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.12 kafka_version: 2.3.1 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.12 kafka_version: 2.4.1 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.12 kafka_version: 2.5.1 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.13 kafka_version: 2.6.3 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.13 kafka_version: 2.7.2 + extra_compose_file: "test/docker-compose-kafka.yml" - scala: 2.13 - kafka_version: 2.8.1 + kafka_version: 2.8.2 + extra_compose_file: "test/docker-compose-kafka.yml" + - scala: 2.13 + kafka_version: 3.0.2 + extra_compose_file: "test/docker-compose-kafka3.yml" + - scala: 2.13 + kafka_version: 3.1.2 + extra_compose_file: "test/docker-compose-kafka3.yml" + - scala: 2.13 + kafka_version: 3.2.3 + extra_compose_file: "test/docker-compose-kafka3.yml" + - scala: 2.13 + kafka_version: 3.3.2 + extra_compose_file: "test/docker-compose-kafka3.yml" + - scala: 2.13 + kafka_version: 3.4.0 + extra_compose_file: "test/docker-compose-kafka3.yml" steps: + - name: Install needed libs + run: | + sudo apt update + sudo apt install -y shellcheck + - name: Checkout uses: actions/checkout@v3 @@ -55,9 +83,56 @@ jobs: id: setup run: | echo BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") >> $GITHUB_OUTPUT + echo DOCKER_IMAGE_VERSION=${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_ENV echo DOCKER_IMAGE_TAG=${{ env.IMAGE_DEST }}:${{ matrix.scala }}-${{ matrix.kafka_version }} >> $GITHUB_OUTPUT - - name: Build and push + - name: Build image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: false + tags: ${{ steps.setup.outputs.DOCKER_IMAGE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + kafka_version=${{ matrix.kafka_version }} + scala_version=${{ matrix.scala }} + vcs_ref=${{ github.sha }} + build_date=${{ steps.setup.outputs.BUILD_DATE }} + + - name: Create test env using docker-compose + run: | + docker-compose -f test/docker-compose.yml -f ${{ matrix.extra_compose_file }} up -d zookeeper kafka_1 kafka_2 + + - name: Perform shellcheck + run: | + echo "Shellcheck main source files" + shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh + cd test + echo "Shellcheck the tests" + shellcheck -x -e SC1090 -s bash *.sh **/*.sh + + - name: Perform tests for image + run: | + cd test + echo "Verify docker image's label" + ./verifyImageLabels.sh + echo "Wait for containers to start" + sleep 5 + # docker-compose logs + docker ps -a + echo "Run tests" + ./runAllTests.sh + echo "End-to-End scenario tests" + cd scenarios + ./runJmxScenario.sh + + - name: Destroy test env using docker-compose + run: | + docker-compose down + + - name: Push image uses: docker/build-push-action@v4 with: context: . @@ -70,4 +145,4 @@ jobs: kafka_version=${{ matrix.kafka_version }} scala_version=${{ matrix.scala }} vcs_ref=${{ github.sha }} - build_date=${{ steps.setup.outputs.BUILD_DATE }} \ No newline at end of file + build_date=${{ steps.setup.outputs.BUILD_DATE }} diff --git a/.travis.yml b/.travis.yml index 04fe8439..3253c94f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,27 +8,56 @@ services: # This version will be also tagged as 'latest' env: global: - - LATEST="2.13-2.8.1" + - LATEST="2.13-3.2.0" # Build recommended versions based on: http://kafka.apache.org/downloads matrix: include: - scala: 2.12 - env: KAFKA_VERSION=2.1.1 + env: + - KAFKA_VERSION=2.1.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.2.2 + env: + - KAFKA_VERSION=2.2.2 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.3.1 + env: + - KAFKA_VERSION=2.3.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.4.1 + env: + - KAFKA_VERSION=2.4.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.12 - env: KAFKA_VERSION=2.5.1 + env: + - KAFKA_VERSION=2.5.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.13 - env: KAFKA_VERSION=2.6.3 + env: + - KAFKA_VERSION=2.6.3 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.13 - env: KAFKA_VERSION=2.7.2 + env: + - KAFKA_VERSION=2.7.2 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml - scala: 2.13 - env: KAFKA_VERSION=2.8.1 + env: + - KAFKA_VERSION=2.8.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka.yml + - scala: 2.13 + env: + - KAFKA_VERSION=3.0.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml + - scala: 2.13 + env: + - KAFKA_VERSION=3.1.1 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml + - scala: 2.13 + env: + - KAFKA_VERSION=3.2.0 + - EXTRA_COMPOSE_FILE=test/docker-compose-kafka3.yml + # Upgrade Docker Engine so we can use buildx before_install: @@ -60,7 +89,7 @@ install: - docker pull confluentinc/cp-kafkacat before_script: - - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2 + - docker-compose -f test/docker-compose.yml -f ${EXTRA_COMPOSE_FILE} up -d zookeeper kafka_1 kafka_2 script: # Shellcheck main source files diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ed2cf9..fa848a48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ Changelog ========= Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date. +20-May-2022 +----------- +- Add support for Kafka `3.0.1`,`3.1.1` and `3.2.0` 09-Apr-2022 ---------- diff --git a/Dockerfile b/Dockerfile index 49668583..eaeab097 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM openjdk:11-jre-slim -ARG kafka_version=2.8.1 +ARG kafka_version=3.2.0 ARG scala_version=2.13 ARG vcs_ref=unspecified ARG build_date=unspecified diff --git a/README.md b/README.md index ca0ce84a..0d9709ee 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ -[![Docker Pulls](https://img.shields.io/docker/pulls/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) -[![Docker Stars](https://img.shields.io/docker/stars/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) -[![](https://images.microbadger.com/badges/version/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own version badge on microbadger.com") -[![](https://images.microbadger.com/badges/image/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own image badge on microbadger.com") -[![Build Status](https://app.travis-ci.com/wurstmeister/kafka-docker.svg?branch=master)](https://app.travis-ci.com/wurstmeister/kafka-docker) +[![Docker Pulls](https://img.shields.io/docker/pulls/ssorriaux/kafka.svg)](https://hub.docker.com/r/ssorriaux/kafka/) +[![Docker Stars](https://img.shields.io/docker/stars/ssorriaux/kafka.svg)](https://hub.docker.com/r/ssorriaux/kafka/) +[![](https://images.microbadger.com/badges/version/ssorriaux/kafka.svg)](https://microbadger.com/images/ssorriaux/kafka "Get your own version badge on microbadger.com") +[![](https://images.microbadger.com/badges/image/ssorriaux/kafka.svg)](https://microbadger.com/images/ssorriaux/kafka "Get your own image badge on microbadger.com") +[![Build Status](https://app.travis-ci.com/ssorriaux/kafka-docker.svg?branch=master)](https://app.travis-ci.com/ssorriaux/kafka-docker) kafka-docker ============ +**Fork of [wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) since no update was provided since May, 2022.** + Dockerfile for [Apache Kafka](http://kafka.apache.org/) -The image is available directly from [Docker Hub](https://hub.docker.com/r/wurstmeister/kafka/) +The image is available directly from [Docker Hub](https://hub.docker.com/r/ssorriaux/kafka/) Tags and releases ----------------- @@ -18,7 +20,7 @@ Tags and releases All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `-`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). To list all available tags: ``` -curl -s https://registry.hub.docker.com/v2/repositories/wurstmeister/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*' +curl -s https://registry.hub.docker.com/v2/repositories/ssorriaux/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*' ``` Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image. @@ -34,7 +36,7 @@ Everytime the image is updated, all tags will be pushed with the latest updates. ## Pre-Requisites - install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/) -- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/wurstmeister/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) +- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in [docker-compose.yml](https://raw.githubusercontent.com/ssorriaux/kafka-docker/master/docker-compose.yml) to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) - if you want to customize any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'``` - Kafka's log4j usage can be customized by adding environment variables prefixed with ```LOG4J_```. These will be mapped to ```log4j.properties```. For example: ```LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER=DEBUG, authorizerAppender``` @@ -56,7 +58,7 @@ Destroy a cluster: ## Note -The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/wurstmeister/kafka-docker/blob/master/docker-compose-single-broker.yml): +The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/StephenSorriaux/kafka-docker/blob/master/docker-compose-single-broker.yml): - ```docker-compose -f docker-compose-single-broker.yml up``` @@ -89,7 +91,7 @@ For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'"` would use a newline to sp ## Advertised hostname -You can configure the advertised hostname in different ways +You can configure the advertised hostname in different ways (Note that as of kafka 3 KAFKA_ADVERTISED_HOST_NAME,KAFKA_ADVERTISED_PORT KAFKA_PORT and KAFKA_HOST_NAME is no longer supported) 1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME``` 2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"``` diff --git a/broker-list.sh b/broker-list.sh index 5c5ee2d7..4648330d 100755 --- a/broker-list.sh +++ b/broker-list.sh @@ -1,5 +1,6 @@ #!/bin/bash CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') -BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) +# We are just interested in ipv4 addresses, so ignore ipv6 addresses +BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | grep '0.0.0.0' | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) echo "${BROKERS//$'\n'/,}" diff --git a/create-topics.sh b/create-topics.sh index 0bacf7b5..21bc83fb 100755 --- a/create-topics.sh +++ b/create-topics.sh @@ -11,15 +11,34 @@ fi start_timeout_exceeded=false count=0 step=10 -while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do - echo "waiting for kafka to be ready" - sleep $step; - count=$((count + step)) - if [ $count -gt $START_TIMEOUT ]; then - start_timeout_exceeded=true - break + +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" +# since 3.0.0 there is no --zookeeper option anymore, so we have to use the +# --bootstrap-server option. +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + if [[ -v KAFKA_LISTENERS ]]; then + PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}' ) + CONNECT_OPTS="--bootstrap-server localhost:${PORT}" + else + CONNECT_OPTS="--bootstrap-server ${BROKER_LIST}" fi -done +else + PORT="$KAFKA_PORT" + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" +fi + +if [[ ! -v PORT ]]; then + while netstat -lnt | awk '$4 ~ /:'"$PORT"'$/ {exit 1}'; do + echo "waiting for kafka to be ready" + sleep $step; + count=$((count + step)) + if [ $count -gt $START_TIMEOUT ]; then + start_timeout_exceeded=true + break + fi + done +fi if $start_timeout_exceeded; then echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" @@ -27,8 +46,6 @@ if $start_timeout_exceeded; then fi # introduced in 0.10. In earlier versions, this will fail because the topic already exists. -# shellcheck disable=SC1091 -source "/usr/bin/versions.sh" if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then KAFKA_0_10_OPTS="--if-not-exists" fi @@ -45,7 +62,7 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ --create \\ - --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ + ${CONNECT_OPTS} \\ --topic ${topicConfig[0]} \\ --partitions ${topicConfig[1]} \\ --replication-factor ${topicConfig[2]} \\ diff --git a/docker-compose-single-broker.yml b/docker-compose-single-broker.yml index 4d8e9f51..e6f45373 100644 --- a/docker-compose-single-broker.yml +++ b/docker-compose-single-broker.yml @@ -9,7 +9,8 @@ services: ports: - "9092:9092" environment: - KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092 + KAFKA_LISTENERS: PLAINTEXT://:9092 KAFKA_CREATE_TOPICS: "test:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: diff --git a/docker-compose.yml b/docker-compose.yml index b4f77b38..73e0d591 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,8 @@ services: - "9092" environment: DOCKER_API_VERSION: 1.22 - KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092 + KAFKA_LISTENERS: PLAINTEXT://:9092 volumes: - /var/run/docker.sock:/var/run/docker.sock restart: unless-stopped diff --git a/start-kafka.sh b/start-kafka.sh index b48280a4..148fa577 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -1,5 +1,8 @@ #!/bin/bash -e +# shellcheck disable=SC1091 +source /usr/bin/versions.sh + # Allow specific kafka versions to perform any unique bootstrap operations OVERRIDE_FILE="/opt/overrides/${KAFKA_VERSION}.sh" if [[ -x "$OVERRIDE_FILE" ]]; then @@ -10,24 +13,45 @@ fi # Store original IFS config, so we can restore it at various stages ORIG_IFS=$IFS -if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then - echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" - exit 1 +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + if [[ -v KAFKA_ADVERTISED_HOST_NAME ]]; then + echo "ERROR: KAFKA_ADVERTISED_HOST_NAME is removed as of kafka 3, remove KAFKA_ADVERTISED_HOST_NAME=$KAFKA_ADVERTISED_HOST_NAME from your config" + exit 1 + fi + if [[ -v KAFKA_ADVERTISED_PORT ]]; then + echo "ERROR: KAFKA_ADVERTISED_PORT is removed as of kafka 3, remove KAFKA_ADVERTISED_PORT=$KAFKA_ADVERTISED_PORT from your config" + exit 1 + fi + if [[ -v KAFKA_PORT ]]; then + echo "ERROR: KAFKA_PORT is removed as of kafka 3, remove KAFKA_PORT=$KAFKA_PORT from your config" + exit 1 + fi + if [[ -v KAFKA_HOST_NAME ]]; then + echo "ERROR: KAFKA_HOST_NAME is removed as of kafka 3, remove KAFKA_HOST_NAME=$KAFKA_HOST_NAME from your config" + exit 1 + fi +else + if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then + echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" + exit 1 + fi + if [[ -z "$KAFKA_PORT" ]]; then + export KAFKA_PORT=9092 + fi fi -if [[ -z "$KAFKA_PORT" ]]; then - export KAFKA_PORT=9092 -fi create-topics.sh & unset KAFKA_CREATE_TOPICS -if [[ -z "$KAFKA_ADVERTISED_PORT" && \ - -z "$KAFKA_LISTENERS" && \ - -z "$KAFKA_ADVERTISED_LISTENERS" && \ - -S /var/run/docker.sock ]]; then - KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) - export KAFKA_ADVERTISED_PORT +if [[ "$MAJOR_VERSION" -lt "3" ]]; then + if [[ -z "$KAFKA_ADVERTISED_PORT" && \ + -z "$KAFKA_LISTENERS" && \ + -z "$KAFKA_ADVERTISED_LISTENERS" && \ + -S /var/run/docker.sock ]]; then + KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) + export KAFKA_ADVERTISED_PORT + fi fi if [[ -z "$KAFKA_BROKER_ID" ]]; then @@ -81,19 +105,30 @@ if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then fi # Try and configure minimal settings or exit with error if there isn't enough information -if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then - if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then - echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS" - exit 1 - elif [[ -z "$HOSTNAME_VALUE" ]]; then - echo "ERROR: No listener or advertised hostname configuration provided in environment." - echo " Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME" - exit 1 - fi +if [[ "$MAJOR_VERSION" -lt "3" ]]; then + if [[ -z "$KAFKA_ADVERTISED_HOST_NAME$KAFKA_LISTENERS" ]]; then + if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then + echo "ERROR: Missing environment variable KAFKA_LISTENERS. Must be specified when using KAFKA_ADVERTISED_LISTENERS" + exit 1 + elif [[ -z "$HOSTNAME_VALUE" ]]; then + echo "ERROR: No listener or advertised hostname configuration provided in environment." + echo " Please define KAFKA_LISTENERS / (deprecated) KAFKA_ADVERTISED_HOST_NAME" + exit 1 + fi - # Maintain existing behaviour - # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined. - export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" + # Maintain existing behaviour + # If HOSTNAME_COMMAND is provided, set that to the advertised.host.name value if listeners are not defined. + export KAFKA_ADVERTISED_HOST_NAME="$HOSTNAME_VALUE" + fi +else + if [[ "$HOSTNAME_VALUE" ]]; then + if [[ -z "$KAFKA_ADVERTISED_LISTENERS" ]]; then + export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$HOSTNAME_VALUE:9092" + if [[ -z "$KAFKA_LISTENERS" ]]; then + export KAFKA_LISTENERS=PLAINTEXT://:9092 + fi + fi + fi fi #Issue newline to config file in case there is not one already diff --git a/test/0.0/test.create-topics-custom-separator.kafka.sh b/test/0.0/test.create-topics-custom-separator.kafka.sh index abd031c1..c26e8131 100755 --- a/test/0.0/test.create-topics-custom-separator.kafka.sh +++ b/test/0.0/test.create-topics-custom-separator.kafka.sh @@ -20,13 +20,25 @@ testCreateTopicsCustomSeparator() { create-topics.sh + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no --zookeeper option anymore, so we have to use the + # --bootstrap-server option with a random broker + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" + else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" + fi + # Loop through each array, validate that topic exists for i in "${!TOPICS[@]}"; do TOPIC=${TOPICS[i]} echo "Validating topic '$TOPIC'" - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") + # shellcheck disable=SC2086 + EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC") if [[ "$EXISTS" != "$TOPIC" ]]; then echo "$TOPIC topic not created" return 1 @@ -36,8 +48,18 @@ testCreateTopicsCustomSeparator() { return 0 } +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" + +# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}') +else + PORT="$KAFKA_PORT" +fi + # mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } +function netstat() { echo "1 2 3 :$PORT"; } export -f netstat testCreateTopicsCustomSeparator diff --git a/test/0.0/test.start-kafka-advertised-host.kafka.sh b/test/0.0/test.start-kafka-advertised-host.kafka.sh index 1791d60b..427df650 100755 --- a/test/0.0/test.start-kafka-advertised-host.kafka.sh +++ b/test/0.0/test.start-kafka-advertised-host.kafka.sh @@ -16,5 +16,14 @@ testAdvertisedHost() { assertAbsent 'advertised.listeners' assertAbsent 'listeners' } +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" -testAdvertisedHost +# since 3.0.0 there is no --zookeeper option anymore, so we have to use the +# --bootstrap-server option with a random broker +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + echo "with kafka from version 3.0.0 'advertised.port' and 'advertised.host.name' are removed, making this test obosolete" + echo "See: https://github.com/apache/kafka/pull/10872" +else + testAdvertisedHost +fi \ No newline at end of file diff --git a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh index 5f0167c3..59703095 100755 --- a/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh +++ b/test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh @@ -3,9 +3,16 @@ source test.functions testKafkaEnv() { + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf" + # since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + + export KAFKA_ADVERTISED_HOST_NAME="testhost" + fi # When the script is invoked source "$START_KAFKA" @@ -22,4 +29,4 @@ testKafkaEnv() { echo " > Set KAFKA_OPTS=$KAFKA_OPTS" } -testKafkaEnv +testKafkaEnv \ No newline at end of file diff --git a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh index df7b4018..720c2f39 100755 --- a/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh +++ b/test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh @@ -3,8 +3,15 @@ source test.functions testKafkaOpts() { + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + export KAFKA_ADVERTISED_HOST_NAME="testhost" + fi # Given required settings are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" + # .. and a CUSTOM_INIT_SCRIPT with spaces export CUSTOM_INIT_SCRIPT="export KAFKA_OPTS=-Djava.security.auth.login.config=/kafka_server_jaas.conf" @@ -20,4 +27,4 @@ testKafkaOpts() { echo " > Set KAFKA_OPTS=$KAFKA_OPTS" } -testKafkaOpts +testKafkaOpts \ No newline at end of file diff --git a/test/0.0/test.start-kafka-host-name.kafka.sh b/test/0.0/test.start-kafka-host-name.kafka.sh index 08dc2113..5aabf205 100755 --- a/test/0.0/test.start-kafka-host-name.kafka.sh +++ b/test/0.0/test.start-kafka-host-name.kafka.sh @@ -8,11 +8,20 @@ testHostnameCommand() { # When the script is invoked source "$START_KAFKA" + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" - # Then the configuration uses the value from the command - assertExpectedConfig 'advertised.host.name=my-host' - assertAbsent 'advertised.listeners' - assertAbsent 'listeners' + # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + # Then the configuration uses the value from the command + assertExpectedConfig 'advertised.listeners=PLAINTEXT://my-host:9092' + assertExpectedConfig 'listeners=PLAINTEXT://:9092' + else + # Then the configuration uses the value from the command + assertExpectedConfig 'advertised.host.name=my-host' + assertAbsent 'advertised.listeners' + assertAbsent 'listeners' + fi } testHostnameCommand diff --git a/test/0.0/test.start-kafka-log4j-config.kafka.sh b/test/0.0/test.start-kafka-log4j-config.kafka.sh index da4ff28e..f1d68614 100755 --- a/test/0.0/test.start-kafka-log4j-config.kafka.sh +++ b/test/0.0/test.start-kafka-log4j-config.kafka.sh @@ -4,7 +4,14 @@ source test.functions testLog4jConfig() { # Given Log4j overrides are provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + + # since 3.0.0 there is no KAFKA_ADVERTISED_HOST_NAME + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + export KAFKA_ADVERTISED_HOST_NAME="testhost" + fi + export LOG4J_LOGGER_KAFKA=DEBUG # When the script is invoked @@ -13,5 +20,4 @@ testLog4jConfig() { # Then the configuration file is correct assertExpectedLog4jConfig "log4j.logger.kafka=DEBUG" } - -testLog4jConfig +testLog4jConfig \ No newline at end of file diff --git a/test/0.0/test.start-kafka-restart.kafka.sh b/test/0.0/test.start-kafka-restart.kafka.sh index 68a25598..6d526c4b 100755 --- a/test/0.0/test.start-kafka-restart.kafka.sh +++ b/test/0.0/test.start-kafka-restart.kafka.sh @@ -3,16 +3,27 @@ source test.functions testRestart() { - # Given a hostname is provided - export KAFKA_ADVERTISED_HOST_NAME="testhost" + # shellcheck disable=SC1091 + source "/usr/bin/versions.sh" + # since 3.0.0 KAFKA_ADVERTISED_HOST_NAME was removed + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + # Given a hostname is provided + export KAFKA_ADVERTISED_HOST_NAME="testhost" + else + export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://testhost:9092" + fi # When the container is restarted (Script invoked multiple times) source "$START_KAFKA" source "$START_KAFKA" - # Then the configuration file only has one instance of the config - assertExpectedConfig 'advertised.host.name=testhost' - assertAbsent 'listeners' + if [[ "$MAJOR_VERSION" -lt "3" ]]; then + # Then the configuration file only has one instance of the config + assertExpectedConfig 'advertised.host.name=testhost' + assertAbsent 'listeners' + else + assertExpectedConfig 'advertised.listeners=PLAINTEXT://testhost:9092' + assertAbsent 'listeners' + fi } - -testRestart +testRestart \ No newline at end of file diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh index b318512f..c8a1a6a1 100755 --- a/test/0.10/test.create-topics.kafka.sh +++ b/test/0.10/test.create-topics.kafka.sh @@ -6,29 +6,49 @@ testCreateTopics() { NOW=$(date +%s) # TOPICS array contains the topic name to create / validate - # CLEANUP array contains the expected cleanup policy configuration for the topic + # CONFIG array contains the expected cleanup policy configuration for the topic TOPICS[0]="default-$NOW" - CLEANUP[0]="" + CONFIG[0]="" TOPICS[1]="compact-$NOW" - CLEANUP[1]="compression.type=snappy,cleanup.policy=compact" + CONFIG[1]=$(echo -e "compression.type=snappy\ncleanup.policy=compact" | sort \ + | tr '\n' ',' \ + | sed 's/,$//') KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh - # Loop through each array, validate that topic exists, and correct cleanup policy is set + + # since 3.0.0 there is no --zookeeper option anymore, so we have to use the + # --bootstrap-server option with a random broker + if [[ "$MAJOR_VERSION" -ge "3" ]]; then + CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)" + else + CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}" + fi + + # Loop through each array, validate that topic exists, and correct configuration is set for i in "${!TOPICS[@]}"; do TOPIC=${TOPICS[i]} echo "Validating topic '$TOPIC'" - EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - POLICY=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --describe --topic "$TOPIC" | awk -F'Configs:' '{print $2}' | xargs) + # shellcheck disable=SC2086 + EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC") + # shellcheck disable=SC2086 + ACTUAL_CONFIG=$(/opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe \ + | cut -d'{' -f1 \ + | grep -oE '(compression.type|cleanup.policy)=([^ ,]+)' \ + | sort \ + | tr '\n' ',' \ + | sed 's/,$//') - RESULT="$EXISTS:$POLICY" - EXPECTED="$TOPIC:${CLEANUP[i]}" + RESULT="$EXISTS:$ACTUAL_CONFIG" + EXPECTED="$TOPIC:${CONFIG[i]}" if [[ "$RESULT" != "$EXPECTED" ]]; then - echo "$TOPIC topic not configured correctly: '$RESULT'" + echo "$TOPIC topic not configured correctly:" + echo " Actual: '$RESULT'" + echo " Expected: '$EXPECTED'" return 1 fi done @@ -36,8 +56,18 @@ testCreateTopics() { return 0 } +# shellcheck disable=SC1091 +source "/usr/bin/versions.sh" + +# since 3.0.0 there is no KAFKA_PORT option anymore, so we need to find the port by inspecting KAFKA_LISTENERS +if [[ "$MAJOR_VERSION" -ge "3" ]]; then + PORT=$(echo "$KAFKA_LISTENERS" | awk -F: '{print $3}') +else + PORT="$KAFKA_PORT" +fi + # mock the netstat call as made by the create-topics.sh script -function netstat() { echo "1 2 3 :$KAFKA_PORT"; } +function netstat() { echo "1 2 3 :$PORT"; } export -f netstat testCreateTopics diff --git a/test/Readme.md b/test/Readme.md index cdc72c19..ba50b714 100644 --- a/test/Readme.md +++ b/test/Readme.md @@ -8,7 +8,10 @@ To execute ``` cd test -docker-compose up -d zookeeper kafka_1 kafka_2 +# if kafka veresion < 3 +docker-compose -f docker-compose.yml -f docker-compose-kafka.yml up -d zookeeper kafka_1 kafka_2 +# if kafka veresion >= 3 +docker-compose -f docker-compose.yml -f docker-compose-kafka3.yml up -d zookeeper kafka_1 kafka_2 ./runAllTests.sh ``` @@ -18,7 +21,7 @@ Run selected tests ### Kafka ``` -docker-compose run --rm kafkatest +docker-compose run -e BROKER_LIST=$(./internal-broker-list.sh) --rm kafkatest ``` ### Kafkacat diff --git a/test/docker-compose-kafka.yml b/test/docker-compose-kafka.yml new file mode 100644 index 00000000..6786c860 --- /dev/null +++ b/test/docker-compose-kafka.yml @@ -0,0 +1,23 @@ +version: '2.1' + +x-kafka-environment-defaults: &kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + +services: + kafka_1: + environment: + <<: *kafka-environment-defaults + + kafka_2: + environment: + <<: *kafka-environment-defaults + + kafkatest: + environment: + <<: *kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 \ No newline at end of file diff --git a/test/docker-compose-kafka3.yml b/test/docker-compose-kafka3.yml new file mode 100644 index 00000000..2fa0a743 --- /dev/null +++ b/test/docker-compose-kafka3.yml @@ -0,0 +1,24 @@ +version: '2.1' + +x-kafka-environment-defaults: &kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + +services: + kafka_1: + environment: + <<: *kafka-environment-defaults + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_1:9092 + + kafka_2: + environment: + <<: *kafka-environment-defaults + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka_2:9092 + + kafkatest: + environment: + <<: *kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 \ No newline at end of file diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 42316de2..ad5dd42f 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,18 +1,12 @@ version: '2.1' x-kafka-defaults: &kafka-defaults - image: wurstmeister/kafka + image: "ssorriaux/kafka:${DOCKER_IMAGE_VERSION}" ports: - "9092" volumes: - /var/run/docker.sock:/var/run/docker.sock -x-kafka-environment-defaults: &kafka-environment-defaults - HOSTNAME_COMMAND: "echo $$(hostname)" - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - services: zookeeper: image: wurstmeister/zookeeper @@ -22,19 +16,16 @@ services: <<: *kafka-defaults container_name: test_kafka_1 environment: - <<: *kafka-environment-defaults KAFKA_BROKER_ID: 1 kafka_2: <<: *kafka-defaults container_name: test_kafka_2 environment: - <<: *kafka-environment-defaults KAFKA_BROKER_ID: 2 kafkatest: - image: wurstmeister/kafka + image: "ssorriaux/kafka:${DOCKER_IMAGE_VERSION}" environment: - KAFKA_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - .:/tests @@ -49,7 +40,7 @@ services: image: confluentinc/cp-kafkacat:5.0.0 environment: - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.8.1} + - KAFKA_VERSION=${KAFKA_VERSION-3.2.0} volumes: - .:/tests working_dir: /tests diff --git a/test/runAllTests.sh b/test/runAllTests.sh index a575b21f..da81e67d 100755 --- a/test/runAllTests.sh +++ b/test/runAllTests.sh @@ -7,12 +7,12 @@ echo "BROKER_LIST=$BROKER_LIST" runAll() { # Tests that require kafka - docker-compose run --rm kafkatest + docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkatest RESULT=$? if [[ $RESULT -eq 0 ]]; then # Tests that require kafkacat - docker-compose run --rm kafkacattest + docker-compose run -e BROKER_LIST="${BROKER_LIST}" --rm kafkacattest RESULT=$? fi diff --git a/test/scenarios/jmx/docker-compose.yml b/test/scenarios/jmx/docker-compose.yml index aba78bf2..b86f4ddf 100644 --- a/test/scenarios/jmx/docker-compose.yml +++ b/test/scenarios/jmx/docker-compose.yml @@ -10,11 +10,10 @@ services: - "9092" - "1099" environment: - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099" JMX_PORT: 1099 volumes: @@ -31,7 +30,6 @@ services: test: image: wurstmeister/kafka environment: - KAFKA_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - .:/scenario diff --git a/test/verifyImageLabels.sh b/test/verifyImageLabels.sh index f59c9985..7821d274 100755 --- a/test/verifyImageLabels.sh +++ b/test/verifyImageLabels.sh @@ -1,17 +1,18 @@ #!/bin/bash -e -VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' wurstmeister/kafka) +VCS_REF=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.vcs-ref"}}' ssorriaux/kafka:"${DOCKER_IMAGE_VERSION}") echo "VCS_REF=$VCS_REF" if [ -z "$VCS_REF" ] || [ "$VCS_REF" = "unspecified" ]; then echo "org.label-schema.vcs-ref is empty or unspecified" exit 1 fi -if ! git cat-file -e "$VCS_REF^{commit}"; then - echo "$VCS_REF Not a valid git commit" - exit 1 -fi +# +# if ! git cat-file -e "$VCS_REF^{commit}"; then +# echo "$VCS_REF Not a valid git commit" +# exit 1 +# fi -BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' wurstmeister/kafka) +BUILD_DATE=$(docker inspect -f '{{ index .Config.Labels "org.label-schema.build-date"}}' ssorriaux/kafka:"${DOCKER_IMAGE_VERSION}") echo "BUILD_DATE=$BUILD_DATE" if ! date -d "$BUILD_DATE"; then echo "$BUILD_DATE Not a valid date"