Add ELK stack to docker-compose files

.env

@@ -15,6 +15,10 @@ OPENSEARCH_IMAGE=opensearchproject/opensearch:2.16.0
 POSTGRES_IMAGE=postgres:16.4
 PROMETHEUS_IMAGE=quay.io/prometheus/prometheus:v2.54.1
 VALKEY_IMAGE=valkey/valkey:8.0-alpine
+ELASTICSEARCH_IMAGE=elastic/elasticsearch:7.17.24
+LOGSTASH_IMAGE=elastic/logstash:7.17.24
+KIBANA_IMAGE=elastic/kibana:7.17.24
+
 # must also update the version arg in ./test/tracetesting/Dockerfile
 TRACETEST_IMAGE=kubeshop/tracetest:v1.5.2
 
@@ -153,3 +157,21 @@ JAEGER_SERVICE_HOST=jaeger
 PROMETHEUS_SERVICE_PORT=9090
 PROMETHEUS_SERVICE_HOST=prometheus
 PROMETHEUS_ADDR=${PROMETHEUS_SERVICE_HOST}:${PROMETHEUS_SERVICE_PORT}
+
+#Elasticsearch
+
+ELASTICSEARCH_SERVICE_PORT=9200
+ELASTICSEARCH_SERVICE_HOST=elasticsearch
+ELASTICSEARCH_ADDR=${ELASTICSEARCH_SERVICE_HOST}:${ELASTICSEARCH_SERVICE_PORT}
+
+#Logstash
+
+LOGSTASH_SERVICE_PORT=5044
+LOGSTASH_SERVICE_HOST=logstash
+LOGSTASH_ADDR=${LOGSTASH_SERVICE_HOST}:${LOGSTASH_SERVICE_PORT}
+
+#Kibana
+
+KIBANA_SERVICE_PORT=5601
+KIBANA_SERVICE_HOST=kibana
+KIBANA_ADDR=${KIBANA_SERVICE_HOST}:${KIBANA_SERVICE_PORT}

docker-compose.minimal.yml

@@ -259,6 +259,12 @@ services:
       - GRAFANA_SERVICE_HOST
       - JAEGER_SERVICE_PORT
       - JAEGER_SERVICE_HOST
+      - ELASTICSEARCH_SERVICE_PORT
+      - ELASTICSEARCH_SERVICE_HOST
+      - LOGSTASH_SERVICE_PORT
+      - LOGSTASH_SERVICE_HOST
+      - KIBANA_SERVICE_PORT
+      - KIBANA_SERVICE_HOST
       - OTEL_COLLECTOR_HOST
       - IMAGE_PROVIDER_HOST
       - IMAGE_PROVIDER_PORT
@@ -582,7 +588,7 @@ services:
         limits:
           memory: 200M
     restart: unless-stopped
-    command: ["--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ]
+    command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ]
     user: 0:0
     volumes:
       - ${HOST_FILESYSTEM}:/hostfs:ro
@@ -601,7 +607,6 @@ services:
       - OTEL_COLLECTOR_HOST
       - OTEL_COLLECTOR_PORT_GRPC
       - OTEL_COLLECTOR_PORT_HTTP
-      - ELASTIC_AGENT_OTEL
 
   # Prometheus
   prometheus:
@@ -655,3 +660,68 @@ services:
     ports:
       - "9200"
     logging: *logging
+
+  # Elasticsearch
+  elasticsearch:
+    image: ${ELASTICSEARCH_IMAGE}
+    container_name: elasticsearch
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    restart: unless-stopped
+    environment:
+     - bootstrap.memory_lock=true
+     - xpack.security.enabled=true
+     - xpack.security.http.ssl.enabled=true
+     - xpack.security.http.ssl.key=certs/es01/es01.key
+     - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+     - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+     - xpack.security.transport.ssl.enabled=true
+     - xpack.security.transport.ssl.key=certs/es01/es01.key
+     - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+     - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+     - xpack.security.transport.ssl.verification_mode=certificate
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "9200"
+    logging: *logging
+
+  #Logstash
+  logstash:
+    image: ${LOGSTASH_IMAGE}
+    container_name: logstash
+    volumes:
+     - certs:/usr/share/logstash/certs
+     - logstashdata01:/usr/share/logstash/data
+     - "./logstash_ingest_data/:/usr/share/logstash/ingest_data/"
+     - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro"
+    deploy:
+      resources:
+        limits:
+          memory: 300M
+    restart: unless-stopped
+    ports:
+      - "${LOGSTASH_SERVICE_PORT}:${LOGSTASH_SERVICE_PORT}"
+    logging: *logging
+
+  # Kibana
+  kibana:
+    image: ${KIBANA_IMAGE}
+    container_name: kibana
+    deploy:
+      resources:
+        limits:
+          memory: 100M
+    restart: unless-stopped
+    environment:
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+    ports:
+      - "${KIBANA_SERVICE_PORT}"
+    logging: *logging

docker-compose.yml

@@ -332,6 +332,12 @@ services:
       - JAEGER_SERVICE_PORT
       - JAEGER_SERVICE_HOST
       - OTEL_COLLECTOR_HOST
+      - ELASTICSEARCH_SERVICE_PORT
+      - ELASTICSEARCH_SERVICE_HOST
+      - LOGSTASH_SERVICE_PORT
+      - LOGSTASH_SERVICE_HOST
+      - KIBANA_SERVICE_PORT
+      - KIBANA_SERVICE_HOST
       - IMAGE_PROVIDER_HOST
       - IMAGE_PROVIDER_PORT
       - OTEL_COLLECTOR_PORT_GRPC
@@ -693,7 +699,7 @@ services:
         limits:
           memory: 200M
     restart: unless-stopped
-    command: ["--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ]
+    command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ]
     user: 0:0
     volumes:
       - ${HOST_FILESYSTEM}:/hostfs:ro
@@ -712,7 +718,6 @@ services:
       - OTEL_COLLECTOR_HOST
       - OTEL_COLLECTOR_PORT_GRPC
       - OTEL_COLLECTOR_PORT_HTTP
-      - ELASTIC_AGENT_OTEL
 
   # Prometheus
   prometheus:
@@ -766,3 +771,100 @@ services:
     ports:
       - "9200"
     logging: *logging
+
+  # Elasticsearch
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_IMAGE}
+    container_name: elasticsearch
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    restart: unless-stopped
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata:/usr/share/elasticsearch/data
+    ports:
+      - "9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert /usr/share/elasticsearch/config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  # Logstash
+  logstash:
+    image: docker.elastic.co/logstash/logstash:${LOGSTASH_IMAGE}
+    container_name: logstash
+    command:
+    - "--config.path=/usr/share/logstash/pipeline"
+    - "--path.data=/usr/share/logstash/data"
+    - "--path.logs=/usr/share/logstash/logs"
+    - "--xpack.monitoring.enabled=false"
+    - "--xpack.security.enabled=true"
+    - "--xpack.security.authc.realms.file.file1.type=file"
+    - "--xpack.security.authc.realms.native.native1.type=native"
+    - "--xpack.security.authc.realms.native.native1.order=0"
+    volumes:
+    - certs:/usr/share/logstash/certs
+    - logstashdata01:/usr/share/logstash/data
+    - ./logstash_ingest_data/:/usr/share/logstash/ingest_data/
+    - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+    restart: unless-stopped
+    ports:
+      - "${LOGSTASH_SERVICE_PORT}:${LOGSTASH_SERVICE_PORT}"
+    logging: *logging
+
+  # Kibana
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${KIBANA_IMAGE}
+    container_name: kibana
+    deploy:
+      resources:
+        limits:
+          memory: 100M
+    restart: unless-stopped
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+    volumes:
+    - certs:/usr/share/kibana/config/certs
+    - kibanadata:/usr/share/kibana/data
+    ports:
+    - "${KIBANA_SERVICE_PORT}:${KIBANA_SERVICE_PORT}"
+    logging: *logging