Extend Compose setup and e2e tests with multiple Kafka providers

.env

@@ -0,0 +1,24 @@
+COMPOSE_FILE="compose.yml:compose.kafka-apache.yml"
+# COMPOSE_FILE="compose.yml:compose.kafka-redpanda.yml"
+# COMPOSE_FILE="compose.yml:compose.kafka-tansu.yml"
+# COMPOSE_FILE="compose.yml:compose.kafka-apache.yml:compose.monitoring.yml"
+
+# Database connection details.
+DB_USERNAME=dtrack
+DB_PASSWORD=dtrack
+DB_NAME=dtrack
+
+# Dependency-Track image tags.
+HYADES_TAG="snapshot-native"
+HYADES_APISERVER_TAG="snapshot"
+HYADES_FRONTEND_TAG="snapshot"
+
+# Infrastructure image tags.
+# Which ones are relevant depends on the compose files being used (see COMPOSE_FILE).
+KAFKA_APACHE_TAG="3.8.0"
+KAFKA_REDPANDA_TAG="v24.2.2"
+KAFKA_TANSU_TAG="latest"
+POSTGRES_TAG="17-alpine"
+
+# Tansu configuration.
+KAFKA_TANSU_DB_NAME="tansu"

.github/workflows/e2e-test.yml

@@ -27,6 +27,13 @@ on:
       hyades-version:
         description: Version of Hyades services to test against
         default: snapshot
+      kafka-provider:
+        description: The Kafka provider to test against
+        type: choice
+        options:
+        - APACHE
+        - APACHE_NATIVE
+        - REDPANDA
 
 permissions: { }
 
@@ -45,8 +52,9 @@ jobs:
         cache: maven
     - name: Test
       env:
-        APISERVER_VERSION: ${{ inputs.apiserver-version || 'snapshot' }}
+        HYADES_APISERVER_VERSION: ${{ inputs.apiserver-version || 'snapshot' }}
         HYADES_VERSION: ${{ inputs.hyades-version || 'snapshot' }}
+        KAFKA_PROVIDER: ${{ inputs.kafka-provider || 'APACHE_NATIVE' }}
         OSSINDEX_USERNAME: ${{ secrets.OSSINDEX_USERNAME }}
         OSSINDEX_TOKEN: ${{ secrets.OSSINDEX_TOKEN }}
       run: mvn -B --no-transfer-progress -pl e2e clean verify -Pe2e-all

.gitignore

@@ -39,7 +39,7 @@ nb-configuration.xml
 *.rej
 
 # Local environment
-.env
+!.env
 
 # Secret key
 secret.key

compose.kafka-apache.yml

@@ -0,0 +1,47 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  kafka:
+    image: "apache/kafka-native:${KAFKA_APACHE_TAG}"
+    environment:
+      CLUSTER_ID: 'RvQwrYegSUCkIPkaiAZQlQ'
+      KAFKA_NODE_ID: "1"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka:29092'
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
+      KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:29092'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "kafka", "9092"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    ports:
+    - "9092:9092"
+    volumes:
+    - "kafka-data:/var/lib/kafka/data"
+    restart: unless-stopped
+
+volumes:
+  kafka-data: { }

compose.kafka-redpanda.yml

@@ -0,0 +1,49 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  kafka:
+    image: "docker.redpanda.com/vectorized/redpanda:${KAFKA_REDPANDA_TAG}"
+    command:
+    - redpanda
+    - start
+    - --smp
+    - '1'
+    - --reserve-memory
+    - 0M
+    - --memory
+    - 512M
+    - --overprovisioned
+    - --node-id
+    - '0'
+    - --kafka-addr
+    - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+    - --advertise-kafka-addr
+    - PLAINTEXT://kafka:29092,OUTSIDE://localhost:9092
+    healthcheck:
+      test: [ "CMD-SHELL", "rpk", "cluster", "health", "--exit-when-healthy" ]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    ports:
+    - "9092:9092"
+    volumes:
+    - "redpanda-data:/var/lib/redpanda/data"
+    restart: unless-stopped
+
+volumes:
+  redpanda-data: { }

compose.kafka-tansu.yml

@@ -0,0 +1,61 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  tansu-init:
+    image: "postgres:${POSTGRES_TAG}"
+    entrypoint: /bin/bash
+    command: >-
+      -c '((psql -d ${KAFKA_TANSU_DB_NAME} -q -c "select 1 from cluster limit 1" >/dev/null 2>/dev/null) && echo "Already initialized")
+      || ((psql -c "CREATE DATABASE ${KAFKA_TANSU_DB_NAME}") && (wget -q -O- https://raw.githubusercontent.com/tansu-io/tansu/refs/heads/main/work-dir/initdb.d/010-schema.sql | psql -d tansu))'
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PGHOST: "postgres"
+      PGUSER: "${DB_USERNAME}"
+      PGPASSWORD: "${DB_PASSWORD}"
+    restart: on-failure
+
+  kafka:
+    image: "ghcr.io/tansu-io/tansu:${KAFKA_TANSU_TAG}"
+    command: >-
+      --kafka-cluster-id RvQwrYegSUCkIPkaiAZQlQ
+      --kafka-node-id 1
+      --kafka-listener-url tcp://0.0.0.0:29092
+      --kafka-advertised-listener-url tcp://kafka:29092
+      --storage-engine pg=postgres://${DB_USERNAME}:${DB_PASSWORD}@postgres/${KAFKA_TANSU_DB_NAME}
+      --work-dir /data
+    depends_on:
+      postgres:
+        condition: service_healthy
+      tansu-init:
+        condition: service_completed_successfully
+    environment:
+      RUST_BACKTRACE: "1"
+      RUST_LOG: "info"
+    healthcheck:
+      test: [ "CMD", "/tansu-server", "-h" ]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    volumes:
+    - "tansu-data:/data"
+    restart: unless-stopped
+
+volumes:
+  tansu-data: { }

compose.monitoring.yml

@@ -0,0 +1,61 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  redpanda-console:
+    image: docker.redpanda.com/vectorized/console:v2.7.0
+    depends_on:
+      kafka:
+        condition: service_healthy
+    environment:
+      CONFIG_FILEPATH: "/tmp/config.yml"
+    ports:
+    - "127.0.0.1:28080:8080"
+    volumes:
+    - "./monitoring/redpanda-console:/tmp/config.yml:ro"
+    - "./proto/src/main/proto:/etc/protos:ro"
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:v2.53.1
+    container_name: dt-prometheus
+    ports:
+    - "127.0.0.1:9090:9090"
+    volumes:
+    - "./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
+    - "prometheus-data:/prometheus"
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana-oss:11.1.3
+    container_name: dt-grafana
+    depends_on:
+    - prometheus
+    environment:
+      GF_SECURITY_ADMIN_USER: "admin"
+      GF_SECURITY_ADMIN_PASSWORD: "admin"
+    ports:
+    - "127.0.0.1:3000:3000"
+    volumes:
+    - "grafana-data:/var/lib/grafana"
+    - "./monitoring/grafana/dashboards:/etc/dashboards:ro"
+    - "./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro"
+    restart: unless-stopped
+
+volumes:
+  grafana-data: { }
+  prometheus-data: { }