diff --git a/.buildkite/benchmark_marathon_pipeline.yml b/.buildkite/benchmark_marathon_pipeline.yml new file mode 100644 index 00000000000..d2f8b657726 --- /dev/null +++ b/.buildkite/benchmark_marathon_pipeline.yml @@ -0,0 +1,11 @@ +agents: + provider: gcp + imageProject: elastic-images-prod + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-16" + diskSizeGb: 100 + diskType: pd-ssd + +steps: + - label: "Benchmark Marathon" + command: .buildkite/scripts/benchmark/marathon.sh \ No newline at end of file diff --git a/.buildkite/health_report_tests_pipeline.yml b/.buildkite/health_report_tests_pipeline.yml new file mode 100644 index 00000000000..84d9bf9d4b1 --- /dev/null +++ b/.buildkite/health_report_tests_pipeline.yml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" + cpu: "2" + memory: "4Gi" + ephemeralStorage: "64Gi" + +steps: + # todo: add a step for 8.latest when Health Report MVP is landed + - label: ":logstash: Health Report Integration tests on main branch." + command: | + source .buildkite/scripts/common/vm-agent.sh + ./.buildkite/scripts/health-report-tests/main.sh + retry: + automatic: + - limit: 3 diff --git a/.buildkite/linux_jdk_matrix_pipeline.yml b/.buildkite/linux_jdk_matrix_pipeline.yml index eb42bd5ee07..65d80d6b42a 100644 --- a/.buildkite/linux_jdk_matrix_pipeline.yml +++ b/.buildkite/linux_jdk_matrix_pipeline.yml @@ -60,20 +60,14 @@ steps: value: "adoptiumjdk_21" - label: "Adoptium JDK 17 (Eclipse Temurin)" value: "adoptiumjdk_17" - - label: "Adoptium JDK 11 (Eclipse Temurin)" - value: "adoptiumjdk_11" - label: "OpenJDK 21" value: "openjdk_21" - label: "OpenJDK 17" value: "openjdk_17" - - label: "OpenJDK 11" - value: "openjdk_11" - label: "Zulu 21" value: "zulu_21" - label: "Zulu 17" value: "zulu_17" - - label: "Zulu 11" - value: "zulu_11" - wait: ~ if: build.source != "schedule" && build.source != "trigger_job" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index a75d5d5b9d3..03d61a9b238 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipeline_slug": "logstash-pull-request-pipeline", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, diff --git a/.buildkite/scripts/benchmark/README.md b/.buildkite/scripts/benchmark/README.md new file mode 100644 index 00000000000..330e7b266b6 --- /dev/null +++ b/.buildkite/scripts/benchmark/README.md @@ -0,0 +1,21 @@ +## Steps to set up GCP instance to run benchmark script +- Create an instance "n2-standard-16" with Ubuntu image +- Install docker + - `sudo snap install docker` + - `sudo usermod -a -G docker $USER` +- Install jq +- Install vault + - `sudo snap install vault` + - `vault login --method github` + - `vault kv get -format json secret/ci/elastic-logstash/benchmark` +- Setup Elasticsearch index mapping and alias with `setup/*` +- Import Kibana dashboard with `save-objects/*` +- Run the benchmark script + - Send data to your own Elasticsearch. Customise `VAULT_PATH="secret/ci/elastic-logstash/your/path"` + - Run the script `main.sh` + - or run in background `nohup bash -x main.sh > log.log 2>&1 &` + +## Notes +- Benchmarks should only be compared using the same hardware setup. +- Please do not send the test metrics to the benchmark cluster. You can set `VAULT_PATH` to send data and metrics to your own server. +- Run `all.sh` as calibration which gives you a baseline of performance in different versions. \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/config/uuid b/.buildkite/scripts/benchmark/config/uuid new file mode 100644 index 00000000000..b6ac241f753 --- /dev/null +++ b/.buildkite/scripts/benchmark/config/uuid @@ -0,0 +1 @@ +f74f1a28-25e9-494f-ba41-ca9f13d4446d \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/core.sh b/.buildkite/scripts/benchmark/core.sh new file mode 100755 index 00000000000..a7bb733713b --- /dev/null +++ b/.buildkite/scripts/benchmark/core.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +set -eo pipefail + +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +CONFIG_PATH="$SCRIPT_PATH/config" +source "$SCRIPT_PATH/util.sh" + +usage() { + echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" + echo "Example: $0 4 {persisted|memory|all} 2 2" + exit 1 +} + +parse_args() { + while [[ "$#" -gt 0 ]]; do + if [ -z "$FB_CNT" ]; then + FB_CNT=$1 + elif [ -z "$QTYPE" ]; then + case $1 in + all | persisted | memory) + QTYPE=$1 + ;; + *) + echo "Error: wrong queue type $1" + usage + ;; + esac + elif [ -z "$CPU" ]; then + CPU=$1 + elif [ -z "$MEM" ]; then + MEM=$1 + else + echo "Error: Too many arguments" + usage + fi + shift + done + + # set default value + # number of filebeat + FB_CNT=${FB_CNT:-4} + # all | persisted | memory + QTYPE=${QTYPE:-all} + CPU=${CPU:-4} + MEM=${MEM:-4} + XMX=$((MEM / 2)) + + IFS=',' + # worker multiplier: 1,2,4 + MULTIPLIERS="${MULTIPLIERS:-1,2,4}" + read -ra MULTIPLIERS <<< "$MULTIPLIERS" + BATCH_SIZES="${BATCH_SIZES:-500}" + read -ra BATCH_SIZES <<< "$BATCH_SIZES" + # tags to json array + read -ra TAG_ARRAY <<< "$TAGS" + JSON_TAGS=$(printf '"%s",' "${TAG_ARRAY[@]}" | sed 's/,$//') + JSON_TAGS="[$JSON_TAGS]" + + IFS=' ' + echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" +} + +get_secret() { + VAULT_PATH=${VAULT_PATH:-secret/ci/elastic-logstash/benchmark} + VAULT_DATA=$(vault kv get -format json $VAULT_PATH) + BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') + BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') + BENCHMARK_ES_PW=$(echo $VAULT_DATA | jq -r '.data.es_pw') + + MONITOR_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.monitor_es_host') + MONITOR_ES_USER=$(echo $VAULT_DATA | jq -r '.data.monitor_es_user') + MONITOR_ES_PW=$(echo $VAULT_DATA | jq -r '.data.monitor_es_pw') +} + +pull_images() { + echo "--- Pull docker images" + + if [[ -n "$LS_VERSION" ]]; then + # pull image if it doesn't exist in local + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" + else + # pull the latest snapshot logstash image + # select the SNAPSHOT artifact with the highest semantic version number + LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) + BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') + ARCH=$(arch) + IMAGE_URL="https://snapshots.elastic.co/${BUILD_ID}/downloads/logstash/logstash-$LS_VERSION-docker-image-$ARCH.tar.gz" + IMAGE_FILENAME="$LS_VERSION.tar.gz" + + echo "Download $LS_VERSION from $IMAGE_URL" + [[ ! -e $IMAGE_FILENAME ]] && curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o "$IMAGE_FILENAME" "$IMAGE_URL" + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker load -i "$IMAGE_FILENAME" + fi + + # pull filebeat image + FB_DEFAULT_VERSION="8.13.4" + FB_VERSION=${FB_VERSION:-$FB_DEFAULT_VERSION} + docker pull "docker.elastic.co/beats/filebeat:$FB_VERSION" +} + +generate_logs() { + FLOG_FILE_CNT=${FLOG_FILE_CNT:-4} + SINGLE_SIZE=524288000 + TOTAL_SIZE="$((FLOG_FILE_CNT * SINGLE_SIZE))" + FLOG_PATH="$SCRIPT_PATH/flog" + mkdir -p $FLOG_PATH + + if [[ ! -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" ]]; then + echo "--- Generate logs in background. log: ${FLOG_FILE_CNT}, each size: 500mb" + docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b $TOTAL_SIZE -p $SINGLE_SIZE + fi +} + +check_logs() { + echo "--- Check log generation" + + local cnt=0 + until [[ -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" || $cnt -gt 600 ]]; do + echo "wait 30s" && sleep 30 + cnt=$((cnt + 30)) + done + + ls -lah $FLOG_PATH +} + +start_logstash() { + LS_CONFIG_PATH=$SCRIPT_PATH/ls/config + mkdir -p $LS_CONFIG_PATH + + cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml + cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml + cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid + + LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} + docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ + -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \ + -e BENCHMARK_ES_HOST="$BENCHMARK_ES_HOST" -e BENCHMARK_ES_USER="$BENCHMARK_ES_USER" -e BENCHMARK_ES_PW="$BENCHMARK_ES_PW" \ + -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ + -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ + -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ + -v $LS_CONFIG_PATH/uuid:/usr/share/logstash/data/uuid:ro \ + docker.elastic.co/logstash/logstash:$LS_VERSION +} + +start_filebeat() { + for ((i = 0; i < FB_CNT; i++)); do + FB_PATH="$SCRIPT_PATH/fb${i}" + mkdir -p $FB_PATH + + cp $CONFIG_PATH/filebeat.yml $FB_PATH/filebeat.yml + + docker run -d --name=fb$i --net=host --user=root \ + -v $FB_PATH/filebeat.yml:/usr/share/filebeat/filebeat.yml \ + -v $SCRIPT_PATH/flog:/usr/share/filebeat/flog \ + docker.elastic.co/beats/filebeat:$FB_VERSION filebeat -e --strict.perms=false + done +} + +capture_stats() { + CURRENT=$(jq -r '.flow.output_throughput.current' $NS_JSON) + local eps_1m=$(jq -r '.flow.output_throughput.last_1_minute' $NS_JSON) + local eps_5m=$(jq -r '.flow.output_throughput.last_5_minutes' $NS_JSON) + local worker_util=$(jq -r '.pipelines.main.flow.worker_utilization.last_1_minute' $NS_JSON) + local worker_concurr=$(jq -r '.pipelines.main.flow.worker_concurrency.last_1_minute' $NS_JSON) + local cpu_percent=$(jq -r '.process.cpu.percent' $NS_JSON) + local heap=$(jq -r '.jvm.mem.heap_used_in_bytes' $NS_JSON) + local non_heap=$(jq -r '.jvm.mem.non_heap_used_in_bytes' $NS_JSON) + local q_event_cnt=$(jq -r '.pipelines.main.queue.events_count' $NS_JSON) + local q_size=$(jq -r '.pipelines.main.queue.queue_size_in_bytes' $NS_JSON) + TOTAL_EVENTS_OUT=$(jq -r '.pipelines.main.events.out' $NS_JSON) + printf "current: %s, 1m: %s, 5m: %s, worker_utilization: %s, worker_concurrency: %s, cpu: %s, heap: %s, non-heap: %s, q_events: %s, q_size: %s, total_events_out: %s\n" \ + $CURRENT $eps_1m $eps_5m $worker_util $worker_concurr $cpu_percent $heap $non_heap $q_event_cnt $q_size $TOTAL_EVENTS_OUT +} + +aggregate_stats() { + local file_glob="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_*.json" + MAX_EPS_1M=$( jqmax '.flow.output_throughput.last_1_minute' "$file_glob" ) + MAX_EPS_5M=$( jqmax '.flow.output_throughput.last_5_minutes' "$file_glob" ) + MAX_WORKER_UTIL=$( jqmax '.pipelines.main.flow.worker_utilization.last_1_minute' "$file_glob" ) + MAX_WORKER_CONCURR=$( jqmax '.pipelines.main.flow.worker_concurrency.last_1_minute' "$file_glob" ) + MAX_Q_EVENT_CNT=$( jqmax '.pipelines.main.queue.events_count' "$file_glob" ) + MAX_Q_SIZE=$( jqmax '.pipelines.main.queue.queue_size_in_bytes' "$file_glob" ) + + AVG_CPU_PERCENT=$( jqavg '.process.cpu.percent' "$file_glob" ) + AVG_VIRTUAL_MEM=$( jqavg '.process.mem.total_virtual_in_bytes' "$file_glob" ) + AVG_HEAP=$( jqavg '.jvm.mem.heap_used_in_bytes' "$file_glob" ) + AVG_NON_HEAP=$( jqavg '.jvm.mem.non_heap_used_in_bytes' "$file_glob" ) +} + +send_summary() { + echo "--- Send summary to Elasticsearch" + + # build json + local timestamp + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") + SUMMARY="{\"timestamp\": \"$timestamp\", \"version\": \"$LS_VERSION\", \"cpu\": \"$CPU\", \"mem\": \"$MEM\", \"workers\": \"$WORKER\", \"batch_size\": \"$BATCH_SIZE\", \"queue_type\": \"$QTYPE\"" + not_empty "$TOTAL_EVENTS_OUT" && SUMMARY="$SUMMARY, \"total_events_out\": \"$TOTAL_EVENTS_OUT\"" + not_empty "$MAX_EPS_1M" && SUMMARY="$SUMMARY, \"max_eps_1m\": \"$MAX_EPS_1M\"" + not_empty "$MAX_EPS_5M" && SUMMARY="$SUMMARY, \"max_eps_5m\": \"$MAX_EPS_5M\"" + not_empty "$MAX_WORKER_UTIL" && SUMMARY="$SUMMARY, \"max_worker_utilization\": \"$MAX_WORKER_UTIL\"" + not_empty "$MAX_WORKER_CONCURR" && SUMMARY="$SUMMARY, \"max_worker_concurrency\": \"$MAX_WORKER_CONCURR\"" + not_empty "$AVG_CPU_PERCENT" && SUMMARY="$SUMMARY, \"avg_cpu_percentage\": \"$AVG_CPU_PERCENT\"" + not_empty "$AVG_HEAP" && SUMMARY="$SUMMARY, \"avg_heap\": \"$AVG_HEAP\"" + not_empty "$AVG_NON_HEAP" && SUMMARY="$SUMMARY, \"avg_non_heap\": \"$AVG_NON_HEAP\"" + not_empty "$AVG_VIRTUAL_MEM" && SUMMARY="$SUMMARY, \"avg_virtual_memory\": \"$AVG_VIRTUAL_MEM\"" + not_empty "$MAX_Q_EVENT_CNT" && SUMMARY="$SUMMARY, \"max_queue_events\": \"$MAX_Q_EVENT_CNT\"" + not_empty "$MAX_Q_SIZE" && SUMMARY="$SUMMARY, \"max_queue_bytes_size\": \"$MAX_Q_SIZE\"" + not_empty "$TAGS" && SUMMARY="$SUMMARY, \"tags\": $JSON_TAGS" + SUMMARY="$SUMMARY}" + + tee summary.json << EOF +{"index": {}} +$SUMMARY +EOF + + # send to ES + local resp + local err_status + resp=$(curl -s -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json") + echo "$resp" + err_status=$(echo "$resp" | jq -r ".errors") + if [[ "$err_status" == "true" ]]; then + echo "Failed to send summary" + exit 1 + fi +} + +# $1: snapshot index +node_stats() { + NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json + + # curl inside container because docker on mac cannot resolve localhost to host network interface + docker exec -i ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null +} + +# $1: index +snapshot() { + node_stats $1 + capture_stats +} + +create_directory() { + NS_DIR="fb${FB_CNT}c${CPU}m${MEM}" # fb4c4m4 + mkdir -p "$SCRIPT_PATH/$NS_DIR" +} + +queue() { + for QTYPE in "persisted" "memory"; do + worker + done +} + +worker() { + for m in "${MULTIPLIERS[@]}"; do + WORKER=$((CPU * m)) + batch + done +} + +batch() { + for BATCH_SIZE in "${BATCH_SIZES[@]}"; do + run_pipeline + stop_pipeline + done +} + +run_pipeline() { + echo "--- Run pipeline. queue type: $QTYPE, worker: $WORKER, batch size: $BATCH_SIZE" + + start_logstash + start_filebeat + docker ps + + echo "(0) sleep 3m" && sleep 180 + snapshot "0" + + for i in {1..8}; do + echo "($i) sleep 30s" && sleep 30 + snapshot "$i" + + # print docker log when ingestion rate is zero + # remove '.' in number and return max val + [[ $(max -g "${CURRENT/./}" "0") -eq 0 ]] && + docker logs fb0 && + docker logs ls + done + + aggregate_stats + send_summary +} + +stop_pipeline() { + echo "--- Stop Pipeline" + + for ((i = 0; i < FB_CNT; i++)); do + docker stop fb$i + docker rm fb$i + done + + docker stop ls + docker rm ls + + curl -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" -X DELETE $BENCHMARK_ES_HOST/_data_stream/logs-generic-default + echo " data stream deleted " + + # TODO: clean page caches, reduce memory fragmentation + # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 +} + +clean_up() { + # stop log generation if it has not done yet + [[ -n $(docker ps | grep flog) ]] && docker stop flog || true + # remove image + docker image rm docker.elastic.co/logstash/logstash:$LS_VERSION +} diff --git a/.buildkite/scripts/benchmark/main.sh b/.buildkite/scripts/benchmark/main.sh index 9d218d772d4..3d615fcb41b 100755 --- a/.buildkite/scripts/benchmark/main.sh +++ b/.buildkite/scripts/benchmark/main.sh @@ -15,9 +15,8 @@ set -eo pipefail # - The script sends a summary of EPS and resource usage to index `benchmark_summary` # ******************************************************* -SCRIPT_PATH="$(cd "$(dirname "$0")"; pwd)" -CONFIG_PATH="$SCRIPT_PATH/config" -source "$SCRIPT_PATH/util.sh" +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +source "$SCRIPT_PATH/core.sh" ## usage: ## main.sh FB_CNT QTYPE CPU MEM @@ -36,272 +35,9 @@ source "$SCRIPT_PATH/util.sh" ## MEM=4 # number of GB for Logstash container ## QTYPE=memory # queue type to test {persisted|memory|all} ## FB_CNT=4 # number of filebeats to use in benchmark -usage() { - echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" - echo "Example: $0 4 {persisted|memory|all} 2 2" - exit 1 -} - -parse_args() { - while [[ "$#" -gt 0 ]]; do - if [ -z "$FB_CNT" ]; then - FB_CNT=$1 - elif [ -z "$QTYPE" ]; then - case $1 in - all | persisted | memory) - QTYPE=$1 - ;; - *) - echo "Error: wrong queue type $1" - usage - ;; - esac - elif [ -z "$CPU" ]; then - CPU=$1 - elif [ -z "$MEM" ]; then - MEM=$1 - else - echo "Error: Too many arguments" - usage - fi - shift - done - - # set default value - # number of filebeat - FB_CNT=${FB_CNT:-4} - # all | persisted | memory - QTYPE=${QTYPE:-all} - CPU=${CPU:-4} - MEM=${MEM:-4} - XMX=$((MEM / 2)) - - IFS=',' - # worker multiplier: 1,2,4 - MULTIPLIERS="${MULTIPLIERS:-1,2,4}" - read -ra MULTIPLIERS <<< "$MULTIPLIERS" - BATCH_SIZES="${BATCH_SIZES:-500}" - read -ra BATCH_SIZES <<< "$BATCH_SIZES" - - IFS=' ' - echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" -} - -get_secret() { - VAULT_PATH=secret/ci/elastic-logstash/benchmark - VAULT_DATA=$(vault kv get -format json $VAULT_PATH) - BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') - BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') - BENCHMARK_ES_PW=$(echo $VAULT_DATA | jq -r '.data.es_pw') - - MONITOR_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.monitor_es_host') - MONITOR_ES_USER=$(echo $VAULT_DATA | jq -r '.data.monitor_es_user') - MONITOR_ES_PW=$(echo $VAULT_DATA | jq -r '.data.monitor_es_pw') -} - -pull_images() { - echo "--- Pull docker images" - - # pull the latest snapshot logstash image - if [[ -n "$LS_VERSION" ]]; then - docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" - else - # select the SNAPSHOT artifact with the highest semantic version number - LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) - BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') - ARCH=$(arch) - IMAGE_URL="https://snapshots.elastic.co/${BUILD_ID}/downloads/logstash/logstash-$LS_VERSION-docker-image-$ARCH.tar.gz" - IMAGE_FILENAME="$LS_VERSION.tar.gz" - - echo "Download $LS_VERSION from $IMAGE_URL" - [[ ! -e $IMAGE_FILENAME ]] && curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o "$IMAGE_FILENAME" "$IMAGE_URL" - [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker load -i "$IMAGE_FILENAME" - fi - - # pull filebeat image - FB_DEFAULT_VERSION="8.13.4" - FB_VERSION=${FB_VERSION:-$FB_DEFAULT_VERSION} - docker pull "docker.elastic.co/beats/filebeat:$FB_VERSION" -} - -generate_logs() { - FLOG_PATH="$SCRIPT_PATH/flog" - mkdir -p $FLOG_PATH - - if [[ ! -e "$FLOG_PATH/log4.log" ]]; then - echo "--- Generate logs in background. log: 5, size: 500mb" - docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b 2621440000 -p 524288000 - fi -} - -check_logs() { - echo "--- Check log generation" - - local cnt=0 - until [[ -e "$FLOG_PATH/log4.log" || $cnt -gt 600 ]]; do - echo "wait 30s" && sleep 30 - cnt=$((cnt + 30)) - done - - ls -lah $FLOG_PATH -} - -start_logstash() { - LS_CONFIG_PATH=$SCRIPT_PATH/ls/config - mkdir -p $LS_CONFIG_PATH - - cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml - cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml - - LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} - docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ - -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \ - -e BENCHMARK_ES_HOST="$BENCHMARK_ES_HOST" -e BENCHMARK_ES_USER="$BENCHMARK_ES_USER" -e BENCHMARK_ES_PW="$BENCHMARK_ES_PW" \ - -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ - -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ - -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ - docker.elastic.co/logstash/logstash:$LS_VERSION -} - -start_filebeat() { - for ((i = 0; i < FB_CNT; i++)); do - FB_PATH="$SCRIPT_PATH/fb${i}" - mkdir -p $FB_PATH - - cp $CONFIG_PATH/filebeat.yml $FB_PATH/filebeat.yml - - docker run -d --name=fb$i --net=host --user=root \ - -v $FB_PATH/filebeat.yml:/usr/share/filebeat/filebeat.yml \ - -v $SCRIPT_PATH/flog:/usr/share/filebeat/flog \ - docker.elastic.co/beats/filebeat:$FB_VERSION filebeat -e --strict.perms=false - done -} - -capture_stats() { - CURRENT=$(jq -r '.flow.output_throughput.current' $NS_JSON) - local eps_1m=$(jq -r '.flow.output_throughput.last_1_minute' $NS_JSON) - local eps_5m=$(jq -r '.flow.output_throughput.last_5_minutes' $NS_JSON) - local worker_util=$(jq -r '.pipelines.main.flow.worker_utilization.last_1_minute' $NS_JSON) - local worker_concurr=$(jq -r '.pipelines.main.flow.worker_concurrency.last_1_minute' $NS_JSON) - local cpu_percent=$(jq -r '.process.cpu.percent' $NS_JSON) - local heap=$(jq -r '.jvm.mem.heap_used_in_bytes' $NS_JSON) - local non_heap=$(jq -r '.jvm.mem.non_heap_used_in_bytes' $NS_JSON) - local q_event_cnt=$(jq -r '.pipelines.main.queue.events_count' $NS_JSON) - local q_size=$(jq -r '.pipelines.main.queue.queue_size_in_bytes' $NS_JSON) - TOTAL_EVENTS_OUT=$(jq -r '.pipelines.main.events.out' $NS_JSON) - printf "current: %s, 1m: %s, 5m: %s, worker_utilization: %s, worker_concurrency: %s, cpu: %s, heap: %s, non-heap: %s, q_events: %s, q_size: %s, total_events_out: %s\n" \ - $CURRENT $eps_1m $eps_5m $worker_util $worker_concurr $cpu_percent $heap $non_heap $q_event_cnt $q_size $TOTAL_EVENTS_OUT -} - -aggregate_stats() { - local file_glob="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_*.json" - MAX_EPS_1M=$( jqmax '.flow.output_throughput.last_1_minute' "$file_glob" ) - MAX_EPS_5M=$( jqmax '.flow.output_throughput.last_5_minutes' "$file_glob" ) - MAX_WORKER_UTIL=$( jqmax '.pipelines.main.flow.worker_utilization.last_1_minute' "$file_glob" ) - MAX_WORKER_CONCURR=$( jqmax '.pipelines.main.flow.worker_concurrency.last_1_minute' "$file_glob" ) - MAX_Q_EVENT_CNT=$( jqmax '.pipelines.main.queue.events_count' "$file_glob" ) - MAX_Q_SIZE=$( jqmax '.pipelines.main.queue.queue_size_in_bytes' "$file_glob" ) - - AVG_CPU_PERCENT=$( jqavg '.process.cpu.percent' "$file_glob" ) - AVG_VIRTUAL_MEM=$( jqavg '.process.mem.total_virtual_in_bytes' "$file_glob" ) - AVG_HEAP=$( jqavg '.jvm.mem.heap_used_in_bytes' "$file_glob" ) - AVG_NON_HEAP=$( jqavg '.jvm.mem.non_heap_used_in_bytes' "$file_glob" ) -} - -send_summary() { - echo "--- Send summary to Elasticsearch" - - timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") - tee summary.json << EOF -{"index": {}} -{"timestamp": "$timestamp", "version": "$LS_VERSION", "cpu": "$CPU", "mem": "$MEM", "workers": "$WORKER", "batch_size": "$BATCH_SIZE", "queue_type": "$QTYPE", "total_events_out": "$TOTAL_EVENTS_OUT", "max_eps_1m": "$MAX_EPS_1M", "max_eps_5m": "$MAX_EPS_5M", "max_worker_utilization": "$MAX_WORKER_UTIL", "max_worker_concurrency": "$MAX_WORKER_CONCURR", "avg_cpu_percentage": "$AVG_CPU_PERCENT", "avg_heap": "$AVG_HEAP", "avg_non_heap": "$AVG_NON_HEAP", "avg_virtual_memory": "$AVG_VIRTUAL_MEM", "max_queue_events": "$MAX_Q_EVENT_CNT", "max_queue_bytes_size": "$MAX_Q_SIZE"} -EOF - curl -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json" - echo "" -} - -# $1: snapshot index -node_stats() { - NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json - - # curl inside container because docker on mac cannot resolve localhost to host network interface - docker exec -it ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null -} - -# $1: index -snapshot() { - node_stats $1 - capture_stats -} - -create_directory() { - NS_DIR="fb${FB_CNT}c${CPU}m${MEM}" # fb4c4m4 - mkdir -p "$SCRIPT_PATH/$NS_DIR" -} - -queue() { - for QTYPE in "persisted" "memory"; do - worker - done -} - -worker() { - for m in "${MULTIPLIERS[@]}"; do - WORKER=$((CPU * m)) - batch - done -} - -batch() { - for BATCH_SIZE in "${BATCH_SIZES[@]}"; do - run_pipeline - stop_pipeline - done -} - -run_pipeline() { - echo "--- Run pipeline. queue type: $QTYPE, worker: $WORKER, batch size: $BATCH_SIZE" - - start_logstash - start_filebeat - docker ps - - echo "(0) sleep 3m" && sleep 180 - snapshot "0" - - for i in {1..8}; do - echo "($i) sleep 30s" && sleep 30 - snapshot "$i" - - # print docker log when ingestion rate is zero - # remove '.' in number and return max val - [[ $(max -g "${CURRENT/./}" "0") -eq 0 ]] && - docker logs fb0 && - docker logs ls - done - - aggregate_stats - send_summary -} - -stop_pipeline() { - echo "--- Stop Pipeline" - - for ((i = 0; i < FB_CNT; i++)); do - docker stop fb$i - docker rm fb$i - done - - docker stop ls - docker rm ls - - curl -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" -X DELETE $BENCHMARK_ES_HOST/_data_stream/logs-generic-default - echo " data stream deleted " - - # TODO: clean page caches, reduce memory fragmentation - # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 -} - +## FLOG_FILE_CNT=4 # number of files to generate for ingestion +## VAULT_PATH=secret/path # vault path point to Elasticsearch credentials. The default value points to benchmark cluster. +## TAGS=test,other # tags with "," separator. main() { parse_args "$@" get_secret @@ -317,8 +53,7 @@ main() { worker fi - # stop log generation if it has not done yet - [[ -n $(docker ps | grep flog) ]] && docker stop flog || true + clean_up } -main "$@" +main "$@" \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/marathon.sh b/.buildkite/scripts/benchmark/marathon.sh new file mode 100755 index 00000000000..eecbc7d8e24 --- /dev/null +++ b/.buildkite/scripts/benchmark/marathon.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -eo pipefail + +# ******************************************************* +# Run benchmark for versions that have flow metrics +# When the hardware changes, run the marathon task to establish a new baseline. +# Usage: +# nohup bash -x all.sh > log.log 2>&1 & +# Accept env vars: +# STACK_VERSIONS=8.15.0,8.15.1,8.16.0-SNAPSHOT # versions to test. It is comma separator string +# ******************************************************* + +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +source "$SCRIPT_PATH/core.sh" + +parse_stack_versions() { + IFS=',' + STACK_VERSIONS="${STACK_VERSIONS:-8.6.0,8.7.0,8.8.0,8.9.0,8.10.0,8.11.0,8.12.0,8.13.0,8.14.0,8.15.0}" + read -ra STACK_VERSIONS <<< "$STACK_VERSIONS" +} + +main() { + parse_stack_versions + parse_args "$@" + get_secret + generate_logs + check_logs + + USER_QTYPE="$QTYPE" + + for V in "${STACK_VERSIONS[@]}" ; do + LS_VERSION="$V" + QTYPE="$USER_QTYPE" + pull_images + create_directory + if [[ $QTYPE == "all" ]]; then + queue + else + worker + fi + done +} + +main "$@" \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md b/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md new file mode 100644 index 00000000000..335acb0ac0c --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md @@ -0,0 +1,5 @@ +## 20240912 +Updated runtime field `release` to return `true` when `version` contains "SNAPSHOT" + +## 20240912 +Initial dashboards \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/README.md b/.buildkite/scripts/benchmark/save-objects/README.md new file mode 100644 index 00000000000..26420761827 --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/README.md @@ -0,0 +1,14 @@ +benchmark_objects.ndjson contains the following resources + +- Dashboards + - daily snapshot + - released versions +- Data Views + - benchmark + - runtime fields + - | Fields Name | Type | Comment | + |--------------|---------------------------------------------------------------------------------------|--------------------------------------------------| + | versions_num | long | convert semantic versioning to number for graph sorting | + | release | boolean | `true` for released version. `false` for snapshot version. It is for graph filtering. | + +To import objects to Kibana, navigate to Stack Management > Save Objects and click Import \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson new file mode 100644 index 00000000000..ef4511357d3 --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson @@ -0,0 +1,31 @@ +{"attributes":{"allowHidden":false,"fieldAttrs":"{\"5m_num\":{\"customLabel\":\"\"},\"avg_heap_num\":{\"customLabel\":\"\"},\"avg_non_heap_num\":{\"customLabel\":\"\"},\"max_worker_utilization_num\":{\"customLabel\":\"\"},\"max_worker_concurrency_num\":{\"customLabel\":\"\"},\"max_queue_events\":{\"customLabel\":\"\"},\"max_queue_events_num\":{\"customLabel\":\"\"},\"total_events_out_num\":{\"customLabel\":\"\"},\"avg_cpu_percentage_num\":{\"customLabel\":\"\"},\"release\":{},\"version_num\":{}}","fieldFormatMap":"{\"5m_num\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"avg_heap_num\":{\"id\":\"number\"},\"avg_non_heap_num\":{\"id\":\"number\"},\"max_worker_utilization_num\":{\"id\":\"number\"},\"max_worker_concurrency_num\":{\"id\":\"number\"},\"max_queue_events\":{\"id\":\"number\"},\"max_queue_events_num\":{\"id\":\"number\"},\"avg_cpu_percentage_num\":{\"id\":\"number\"}}","fields":"[{\"name\":\"5m_num\",\"script\":\"return Integer.parseInt(doc['max_eps_5m.keyword'].value)\",\"lang\":\"painless\",\"type\":\"number\",\"scripted\":true,\"customLabel\":\"\"}]","name":"benchmark","runtimeFieldMap":"{\"release\":{\"type\":\"boolean\",\"script\":{\"source\":\"def version = doc['version.keyword'].value;\\nif (version.contains(\\\"SNAPSHOT\\\")) {\\n\\temit(false);\\n}\\nelse {\\n\\temit(true);\\n}\"}},\"version_num\":{\"type\":\"long\",\"script\":{\"source\":\"if (doc.containsKey('version.keyword') && !doc['version.keyword'].empty) {\\n // Get the version string\\n String version = doc['version.keyword'].value;\\n \\n // Remove any suffix after the patch number (e.g., -SNAPSHOT)\\n int dashIndex = version.indexOf('-');\\n if (dashIndex != -1) {\\n version = version.substring(0, dashIndex);\\n }\\n \\n // Find positions of dots\\n int firstDot = version.indexOf('.');\\n int secondDot = version.indexOf('.', firstDot + 1);\\n \\n // Extract major, minor, and patch parts of the version\\n int major = Integer.parseInt(version.substring(0, firstDot));\\n int minor = Integer.parseInt(version.substring(firstDot + 1, secondDot));\\n int patch = Integer.parseInt(version.substring(secondDot + 1));\\n \\n // Combine into a sortable numeric value\\n emit(major * 10000 + minor * 100 + patch);\\n}\\n\\nemit(0);\"}}}","sourceFilters":"[]","timeFieldName":"timestamp","title":"benchmark_*"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"c5493557-3fd6-4929-8dc2-40d248303942","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-09-12T20:34:35.037Z","updated_by":"u_4132377203_cloud","version":"WzE1NjYsOF0="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"999c9564-cba1-4f3b-af99-ccbacb8e034f":{"columnOrder":["a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","9ddc0f02-5530-4930-99bd-2a914ada1b8f","be033d18-2c04-44f2-b403-0b88d888143d"],"columns":{"9ddc0f02-5530-4930-99bd-2a914ada1b8f":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e":{"dataType":"string","isBucketed":true,"label":"Top values of workers.keyword + 1 other","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"be033d18-2c04-44f2-b403-0b88d888143d","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"multi_terms"},"secondaryFields":["queue_type.keyword"],"size":100},"scale":"ordinal","sourceField":"workers.keyword"},"be033d18-2c04-44f2-b403-0b88d888143d":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["be033d18-2c04-44f2-b403-0b88d888143d"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["4","memory"]]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["8","memory"]]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","memory"]]},"touched":false},{"color":{"colorCode":"#5e6c94","type":"colorCode"},"rule":{"type":"matchExactly","values":[["8","persisted"]]},"touched":true},{"color":{"colorIndex":4,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","persisted"]]},"touched":false},{"color":{"colorCode":"#a0df7b","type":"colorCode"},"rule":{"type":"matchExactly","values":[["4","persisted"]]},"touched":true}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"999c9564-cba1-4f3b-af99-ccbacb8e034f","layerType":"data","seriesType":"line","splitAccessor":"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","xAccessor":"9ddc0f02-5530-4930-99bd-2a914ada1b8f"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"5m eps"}},"title":"Release EPS by worker & queue","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:29.581Z","id":"2c017667-7403-402c-b027-3cc790945a27","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:29.581Z","version":"WzEzOTMsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[MQ][R] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:13.413Z","id":"9546a73b-5937-4d3d-850f-30625d6e1405","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:13.413Z","version":"WzEzOTEsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"2786f33f-2ad1-4722-8706-9cba9613662e","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ][R] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:00.945Z","id":"6fba7dff-2564-450b-b885-8c6cf97fa727","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"2786f33f-2ad1-4722-8706-9cba9613662e","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:00.945Z","version":"WzEzODksN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"368034aa-1121-464d-8aed-8a4312274fa8","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[MQ][R] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:52:50.960Z","id":"6295535b-736d-4704-99d3-1cc0c4bb4cb4","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"368034aa-1121-464d-8aed-8a4312274fa8","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:52:50.960Z","version":"WzEzODcsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"columns":{"16dcf8b0-378f-48df-8bb3-f537324eea46":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"2c32a023-462c-4205-b462-6a87182732e4":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"0586bc7c-4662-41bd-8e6d-2337477ca621","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":"release : true "},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[MQ][R] memory by worker ","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:52:40.740Z","id":"a577558f-f9a6-4c86-b562-7618b2c71db0","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"0586bc7c-4662-41bd-8e6d-2337477ca621","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:52:40.740Z","version":"WzEzODUsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"d556c1f1-16ab-45de-adda-366de4703635","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[PQ][R] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:52:31.089Z","id":"1872e84f-56f2-4a17-9426-3d25d6e03931","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"d556c1f1-16ab-45de-adda-366de4703635","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:52:31.089Z","version":"WzEzODQsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"a22f82f6-c7b8-4e06-b2f5-69e25c5aefc4","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[PQ][R] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:52.744Z","id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"a22f82f6-c7b8-4e06-b2f5-69e25c5aefc4","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:52.744Z","version":"WzEzODAsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_queue_events","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_queue_events"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"48ba7011-43c1-48fc-bc36-858bcec74547","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"events in queue"}},"title":"[PQ][R] events in queue by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:43.703Z","id":"d79b8742-6284-4842-ac44-65640d090179","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"48ba7011-43c1-48fc-bc36-858bcec74547","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:43.703Z","version":"WzEzNzgsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"b16664e5-7dda-45f4-853a-0ae2a0e0a692","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[PQ][R] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:33.831Z","id":"64d0f8a7-afe3-4443-88e1-7551d666fee8","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"b16664e5-7dda-45f4-853a-0ae2a0e0a692","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:33.831Z","version":"WzEzNzcsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"columns":{"16dcf8b0-378f-48df-8bb3-f537324eea46":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"2c32a023-462c-4205-b462-6a87182732e4":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"9f57d841-9266-4f58-af56-7fa5e40d8da7","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":"release : true "},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[PQ][R] memory by worker ","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:25.955Z","id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"9f57d841-9266-4f58-af56-7fa5e40d8da7","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:25.955Z","version":"WzEzNzYsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"999c9564-cba1-4f3b-af99-ccbacb8e034f":{"columnOrder":["a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","9ddc0f02-5530-4930-99bd-2a914ada1b8f","be033d18-2c04-44f2-b403-0b88d888143d"],"columns":{"9ddc0f02-5530-4930-99bd-2a914ada1b8f":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e":{"dataType":"string","isBucketed":true,"label":"Top values of workers.keyword + 1 other","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"be033d18-2c04-44f2-b403-0b88d888143d","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"multi_terms"},"secondaryFields":["queue_type.keyword"],"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"be033d18-2c04-44f2-b403-0b88d888143d":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"release","index":"0e2f84c5-7f78-4388-ad55-1230425efb24","key":"release","negate":false,"params":{"query":false},"type":"phrase"},"query":{"match_phrase":{"release":false}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["be033d18-2c04-44f2-b403-0b88d888143d"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["4","memory"]]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["8","memory"]]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","memory"]]},"touched":false},{"color":{"colorCode":"#5e6c94","type":"colorCode"},"rule":{"type":"matchExactly","values":[["8","persisted"]]},"touched":true},{"color":{"colorIndex":4,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","persisted"]]},"touched":false},{"color":{"colorCode":"#a0df7b","type":"colorCode"},"rule":{"type":"matchExactly","values":[["4","persisted"]]},"touched":true}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"999c9564-cba1-4f3b-af99-ccbacb8e034f","layerType":"data","seriesType":"line","splitAccessor":"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","xAccessor":"9ddc0f02-5530-4930-99bd-2a914ada1b8f"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"5m eps"}},"title":"Daily EPS by worker & queue","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"0e2f84c5-7f78-4388-ad55-1230425efb24","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzczOCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"columns":{"16dcf8b0-378f-48df-8bb3-f537324eea46":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"2c32a023-462c-4205-b462-6a87182732e4":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"ba0999ad-bf15-429c-ba76-93b1ff919cea","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[MQ] memory by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"ba0999ad-bf15-429c-ba76-93b1ff919cea","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0NCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"8e8a8743-f0a4-47ed-a4f0-21e21971e264","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[MQ] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"8cbdba7b-e921-42b9-b55b-da567717e259","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"8e8a8743-f0a4-47ed-a4f0-21e21971e264","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Nyw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_queue_events","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_queue_events"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"738fa511-8000-4f6a-b892-4a767bd733ab","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"events in queue"}},"title":"[PQ] events in queue by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"7a2c9e96-8e48-4546-b736-065757a52c47","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"738fa511-8000-4f6a-b892-4a767bd733ab","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Myw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"6dab2bf1-e8cd-4d30-99d6-4e8038c8d19b","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[PQ] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"daa3722a-cab7-49ba-b5bb-033e81133096","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"6dab2bf1-e8cd-4d30-99d6-4e8038c8d19b","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0MSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"columns":{"0ad565fc-0a55-492b-ba69-1be57f501e13":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"16dcf8b0-378f-48df-8bb3-f537324eea46":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"962f0beb-0dd8-4449-a652-878d0d2c63e8","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[PQ] memory by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"4d1f7bef-c184-4457-b159-465128d2d70b","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"962f0beb-0dd8-4449-a652-878d0d2c63e8","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzczOSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[PQ] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Miw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Niw3XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":false},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":false}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":14,\"i\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},\"panelIndex\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\",\"embeddableConfig\":{\"enhancements\":{},\"attributes\":{\"title\":\"Daily EPS by queue & worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"seriesType\":\"line\",\"xAccessor\":\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"splitAccessor\":\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"accessors\":[\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#207f8c\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#0055fc\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#99e6e8\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#c3513a\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#fd1900\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#e88888\"},\"touched\":true}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}}}],\"yTitle\":\"5m eps\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"999c9564-cba1-4f3b-af99-ccbacb8e034f\":{\"columns\":{\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\":{\"label\":\"Top values of workers.keyword + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":6,\"orderBy\":{\"type\":\"column\",\"columnId\":\"be033d18-2c04-44f2-b403-0b88d888143d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"queue_type.keyword\"]}},\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"be033d18-2c04-44f2-b403-0b88d888143d\":{\"label\":\"Median of 5m_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"5m_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"type\":\"index-pattern\"}],\"type\":\"lens\"}},\"panelRefName\":\"panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":14,\"w\":16,\"h\":14,\"i\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\"},\"panelIndex\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3fc59b3e-512e-4a47-baa4-e89e4978b685\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":14,\"w\":16,\"h\":14,\"i\":\"6294036c-7c92-4e51-990d-ace6af0de939\"},\"panelIndex\":\"6294036c-7c92-4e51-990d-ace6af0de939\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6294036c-7c92-4e51-990d-ace6af0de939\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":28,\"w\":16,\"h\":8,\"i\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\"},\"panelIndex\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"PQ max q events by worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":true,\"showSingleSeries\":true,\"legendSize\":\"small\",\"shouldTruncate\":true,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"left\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yTitle\":\"events in queue\",\"hideEndzones\":false,\"showCurrentTimeMarker\":false,\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"seriesType\":\"line\",\"splitAccessor\":\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"accessors\":[\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"4\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":0},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"8\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":1},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"16\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":2},\"touched\":false}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"9542ed03-9691-42a3-b5c2-407bcaab221c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"index\":\"599131c3-0cd9-4a1f-b9da-9894538d58cc\",\"key\":\"queue_type.keyword\",\"field\":\"queue_type.keyword\",\"params\":{\"query\":\"persisted\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"queue_type.keyword\":\"persisted\"}},\"$state\":{\"store\":\"appState\"}}],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb818eba-fa75-45d1-a9ee-6619eecc9280\":{\"columns\":{\"0da92b64-fe48-439e-9b3a-47775ce1a149\":{\"label\":\"workers\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":false},\"customLabel\":true},\"9542ed03-9691-42a3-b5c2-407bcaab221c\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\":{\"label\":\"Median of max_queue_events_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"max_queue_events_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"9542ed03-9691-42a3-b5c2-407bcaab221c\",\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"type\":\"index-pattern\"}],\"type\":\"lens\"},\"enhancements\":{}},\"panelRefName\":\"panel_77d40890-d175-436f-aac3-7bcc9f7beacc\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\"},\"panelIndex\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8bd920f9-5be7-4481-b733-ba83bdd127d5\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":28,\"w\":16,\"h\":16,\"i\":\"23581637-2ff1-4cae-a839-345620c3d180\"},\"panelIndex\":\"23581637-2ff1-4cae-a839-345620c3d180\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_23581637-2ff1-4cae-a839-345620c3d180\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":28,\"w\":16,\"h\":16,\"i\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\"},\"panelIndex\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_c7ab44aa-4607-40b7-b268-d69136bfdcde\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":14,\"w\":16,\"h\":14,\"i\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\"},\"panelIndex\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_01628fc9-19c2-4c65-bc9f-3f284f77c309\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-2M","timeRestore":true,"timeTo":"now","title":"Logstash daily benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:44:20.764Z","id":"9ef273e3-bee2-48bb-9976-7edc19454b93","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","name":"3fc59b3e-512e-4a47-baa4-e89e4978b685:panel_3fc59b3e-512e-4a47-baa4-e89e4978b685","type":"lens"},{"id":"8cbdba7b-e921-42b9-b55b-da567717e259","name":"6294036c-7c92-4e51-990d-ace6af0de939:panel_6294036c-7c92-4e51-990d-ace6af0de939","type":"lens"},{"id":"7a2c9e96-8e48-4546-b736-065757a52c47","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:panel_77d40890-d175-436f-aac3-7bcc9f7beacc","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"daa3722a-cab7-49ba-b5bb-033e81133096","name":"8bd920f9-5be7-4481-b733-ba83bdd127d5:panel_8bd920f9-5be7-4481-b733-ba83bdd127d5","type":"lens"},{"id":"4d1f7bef-c184-4457-b159-465128d2d70b","name":"23581637-2ff1-4cae-a839-345620c3d180:panel_23581637-2ff1-4cae-a839-345620c3d180","type":"lens"},{"id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","name":"c7ab44aa-4607-40b7-b268-d69136bfdcde:panel_c7ab44aa-4607-40b7-b268-d69136bfdcde","type":"lens"},{"id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","name":"01628fc9-19c2-4c65-bc9f-3f284f77c309:panel_01628fc9-19c2-4c65-bc9f-3f284f77c309","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2024-09-09T13:44:20.764Z","version":"WzE0ODksOF0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":true},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":true}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":19,\"i\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\"},\"panelIndex\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_329684df-6ec4-4973-8c96-bc91eaa23fe9\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":19,\"w\":16,\"h\":17,\"i\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},\"panelIndex\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":19,\"w\":16,\"h\":17,\"i\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\"},\"panelIndex\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_485385cb-cd2b-408b-a119-5e7b15c15eeb\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":19,\"w\":16,\"h\":17,\"i\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},\"panelIndex\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":36,\"w\":16,\"h\":16,\"i\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\"},\"panelIndex\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2934659f-462a-41b1-bdc0-344a7a4a2267\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":36,\"w\":16,\"h\":16,\"i\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},\"panelIndex\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\"},\"panelIndex\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_99c4094f-8906-4ded-b56f-8e20107a8f3b\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":44,\"w\":16,\"h\":8,\"i\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\"},\"panelIndex\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_a8dd132b-9212-4c33-83ef-19ec5152448e\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-1y","timeRestore":true,"timeTo":"now","title":"Logstash release benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:43:43.023Z","id":"901317cf-f97a-4d20-8420-53f0bf67c606","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"2c017667-7403-402c-b027-3cc790945a27","name":"329684df-6ec4-4973-8c96-bc91eaa23fe9:panel_329684df-6ec4-4973-8c96-bc91eaa23fe9","type":"lens"},{"id":"a577558f-f9a6-4c86-b562-7618b2c71db0","name":"68db206a-6de8-40a8-bd6c-b82d78a41ac7:panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7","type":"lens"},{"id":"9546a73b-5937-4d3d-850f-30625d6e1405","name":"485385cb-cd2b-408b-a119-5e7b15c15eeb:panel_485385cb-cd2b-408b-a119-5e7b15c15eeb","type":"lens"},{"id":"6fba7dff-2564-450b-b885-8c6cf97fa727","name":"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce:panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce","type":"lens"},{"id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","name":"2934659f-462a-41b1-bdc0-344a7a4a2267:panel_2934659f-462a-41b1-bdc0-344a7a4a2267","type":"lens"},{"id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","name":"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2:panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2","type":"lens"},{"id":"d79b8742-6284-4842-ac44-65640d090179","name":"99c4094f-8906-4ded-b56f-8e20107a8f3b:panel_99c4094f-8906-4ded-b56f-8e20107a8f3b","type":"lens"},{"id":"1872e84f-56f2-4a17-9426-3d25d6e03931","name":"a8dd132b-9212-4c33-83ef-19ec5152448e:panel_a8dd132b-9212-4c33-83ef-19ec5152448e","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2024-09-09T13:43:43.023Z","version":"WzE0OTAsOF0="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[PQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"07700ebe-b8c9-498f-9a91-30efb0c48784","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0MCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"4e0f2401-369e-4818-8a1f-52b64ff175df","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[MQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"29ebb6b1-bee6-4f15-b3dc-aac083d5d2f5","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"4e0f2401-369e-4818-8a1f-52b64ff175df","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0NSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"string","isBucketed":true,"label":"Top 100 values of version.keyword","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","negate":false,"params":[{"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}},{"meta":{"alias":null,"disabled":false,"field":"release","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"release","negate":false,"params":{"query":true},"type":"phrase"},"query":{"match_phrase":{"release":true}}}],"relation":"AND","type":"combined"},"query":{}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[MQ][R] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"26ec195e-e4a9-4417-be97-6d1fbc6b63c0","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0OSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"string","isBucketed":true,"label":"Top 100 values of version.keyword","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"da1edddb-a868-406c-b3a3-e02cf1069e50","negate":false,"params":[{"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}},{"meta":{"alias":null,"disabled":false,"field":"release","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"release","negate":false,"params":{"query":true},"type":"phrase"},"query":{"match_phrase":{"release":true}}}],"relation":"AND","type":"combined"},"query":{}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[PQ][R] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"bb65f9af-1246-4e75-8b61-dea93d920da8","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"da1edddb-a868-406c-b3a3-e02cf1069e50","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1MCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"8171fdc1-2ff4-4f45-8796-dffd31749c63","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"max 5m_eps"}},"title":"[MQ] EPS by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"5c7f9518-5dac-4a3f-b8e1-6ece4135a8be","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1MSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"4e1e2474-ea84-47f5-bb34-0bda229814df","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"max 5m_eps"}},"title":"[PQ] EPS by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f24fc69c-7714-40d3-b34e-3d90fa9b7023","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"4e1e2474-ea84-47f5-bb34-0bda229814df","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1Miw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"637adce1-2cf3-4869-a35d-ead85c340ab9","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[PQ] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"fc548f47-c5ab-491f-a234-4f24ce439d55","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"637adce1-2cf3-4869-a35d-ead85c340ab9","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1Myw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"26183784-8280-42c4-ae7c-71c4e9a5908c","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[MQ] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"b3c44ade-7f2f-497a-bc9b-f4b53695d975","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"26183784-8280-42c4-ae7c-71c4e9a5908c","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1NCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"904fa8c5-7ce1-45ec-bc0d-a52929cabaa6":{"columnOrder":["3dfc74bb-0629-4809-8416-282f6924ecf6","2e3311e8-099f-48bc-81e8-f8dd643cee72","c62ffb65-c24f-49fe-b8ed-683257e60808"],"columns":{"2e3311e8-099f-48bc-81e8-f8dd643cee72":{"dataType":"number","isBucketed":false,"label":"Moving average of Last value of 5m_num","operationType":"moving_average","params":{"window":5},"references":["c62ffb65-c24f-49fe-b8ed-683257e60808"],"scale":"ratio"},"3dfc74bb-0629-4809-8416-282f6924ecf6":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"c62ffb65-c24f-49fe-b8ed-683257e60808":{"dataType":"number","filter":{"language":"kuery","query":"\"5m_num\": *"},"isBucketed":false,"label":"Last value of 5m_num","operationType":"last_value","params":{"showArrayValues":true,"sortField":"timestamp"},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"None","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["2e3311e8-099f-48bc-81e8-f8dd643cee72"],"colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"904fa8c5-7ce1-45ec-bc0d-a52929cabaa6","layerType":"data","position":"top","seriesType":"bar","showGridlines":false,"xAccessor":"3dfc74bb-0629-4809-8416-282f6924ecf6"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"bar","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide"}},"title":"logstash_5m_eps","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-07-08T16:56:26.170Z","id":"af3a2cb0-138f-4b4a-aaf8-49ff4325386d","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-904fa8c5-7ce1-45ec-bc0d-a52929cabaa6","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-07-08T16:56:26.170Z","version":"WzQzLDRd"} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":30,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/setup/alias b/.buildkite/scripts/benchmark/setup/alias new file mode 100644 index 00000000000..efa658a4d22 --- /dev/null +++ b/.buildkite/scripts/benchmark/setup/alias @@ -0,0 +1,6 @@ +POST /_aliases +{ + "actions": [ + { "add": { "index": "benchmark_summary_v2", "alias": "benchmark_summary" } } + ] +} \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/setup/benchmark_summary_v2 b/.buildkite/scripts/benchmark/setup/benchmark_summary_v2 new file mode 100644 index 00000000000..57c80f8c7b3 --- /dev/null +++ b/.buildkite/scripts/benchmark/setup/benchmark_summary_v2 @@ -0,0 +1,179 @@ +PUT /benchmark_summary_v2/_mapping +{ + "properties": { + "avg_cpu_percentage": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "avg_heap": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "avg_non_heap": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "avg_virtual_memory": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "batch_size": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "cpu": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_eps_1m": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_eps_5m": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_queue_bytes_size": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_queue_events": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_worker_concurrency": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_worker_utilization": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "mem": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "queue_type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tag": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "timestamp": { + "type": "date" + }, + "total_events_out": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "workers": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tags" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } +} diff --git a/.buildkite/scripts/benchmark/util.sh b/.buildkite/scripts/benchmark/util.sh index d7e714e0ba6..ac036d59e68 100755 --- a/.buildkite/scripts/benchmark/util.sh +++ b/.buildkite/scripts/benchmark/util.sh @@ -30,3 +30,12 @@ jqavg() { jqmax() { jq -r "$1 | select(. != null)" $2 | jq -s . | jq 'max' } + +# return true if $1 is non empty and not "null" +not_empty() { + if [[ -n "$1" && "$1" != "null" ]]; then + return 0 + else + return 1 + fi +} \ No newline at end of file diff --git a/.buildkite/scripts/dra/common.sh b/.buildkite/scripts/dra/common.sh index abdf461c1c2..6618c6314ec 100755 --- a/.buildkite/scripts/dra/common.sh +++ b/.buildkite/scripts/dra/common.sh @@ -29,7 +29,7 @@ function save_docker_tarballs { # Since we are using the system jruby, we need to make sure our jvm process # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 -export JRUBY_OPTS="-J-Xmx1g" +export JRUBY_OPTS="-J-Xmx2g" # Extract the version number from the version.yml file # e.g.: 8.6.0 diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh index 598da84bbb7..67c6ce895d9 100755 --- a/.buildkite/scripts/dra/publish.sh +++ b/.buildkite/scripts/dra/publish.sh @@ -15,8 +15,9 @@ RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\. if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then RELEASE_BRANCH=$RELEASE_VER else - RELEASE_BRANCH=main + RELEASE_BRANCH="${BUILDKITE_BRANCH:="main"}" fi +echo "RELEASE BRANCH: $RELEASE_BRANCH" if [ -n "$VERSION_QUALIFIER_OPT" ]; then # Qualifier is passed from CI as optional field and specify the version postfix diff --git a/.buildkite/scripts/health-report-tests/main.sh b/.buildkite/scripts/health-report-tests/main.sh new file mode 100755 index 00000000000..c31a4b120e3 --- /dev/null +++ b/.buildkite/scripts/health-report-tests/main.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +# TODO: +# if branch is specified with X.Y, pull branches from ACTIVE_BRANCHES_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/branches.json", parse and use +# build Logstash from specificed (ex: 8.x -> translates to 8.latest, 8.16) branch, defaults to main +# install requirements of the python package and run main.py + + diff --git a/.buildkite/scripts/snyk/report.sh b/.buildkite/scripts/snyk/report.sh index 7dab7c5e357..0d666f777cc 100755 --- a/.buildkite/scripts/snyk/report.sh +++ b/.buildkite/scripts/snyk/report.sh @@ -4,12 +4,6 @@ set -e TARGET_BRANCHES=("main") -install_java() { - # TODO: let's think about using BK agent which has Java installed - # Current caveat is Logstash BK agent doesn't support docker operatioins in it - sudo apt update && sudo apt install -y openjdk-17-jdk && sudo apt install -y openjdk-17-jre -} - # Resolves the branches we are going to track resolve_latest_branches() { source .buildkite/scripts/snyk/resolve_stack_version.sh @@ -55,7 +49,6 @@ report() { ./snyk monitor --prune-repeated-subdependencies --all-projects --org=logstash --remote-repo-url="$REMOTE_REPO_URL" --target-reference="$REMOTE_REPO_URL" --detection-depth=6 --exclude=qa,tools,devtools,requirements.txt --project-tags=branch="$TARGET_BRANCH",git_head="$GIT_HEAD" || : } -install_java resolve_latest_branches download_auth_snyk @@ -72,69 +65,3 @@ do echo "$TARGET_BRANCH branch doesn't exist." fi done - -# Scan Logstash docker images and report -REPOSITORY_BASE_URL="docker.elastic.co/logstash/" - -report_docker_image() { - image=$1 - project_name=$2 - platform=$3 - echo "Reporting $image to Snyk started..." - docker pull "$image" - if [[ $platform != null ]]; then - ./snyk container monitor "$image" --org=logstash --platform="$platform" --project-name="$project_name" --project-tags=version="$version" || : - else - ./snyk container monitor "$image" --org=logstash --project-name="$project_name" --project-tags=version="$version" || : - fi -} - -report_docker_images() { - version=$1 - echo "Version value: $version" - - image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT" - snyk_project_name="logstash-$version-SNAPSHOT" - report_docker_image "$image" "$snyk_project_name" - - image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT" - snyk_project_name="logstash-oss-$version-SNAPSHOT" - report_docker_image "$image" "$snyk_project_name" - - image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT-arm64" - snyk_project_name="logstash-$version-SNAPSHOT-arm64" - report_docker_image "$image" "$snyk_project_name" "linux/arm64" - - image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT-amd64" - snyk_project_name="logstash-$version-SNAPSHOT-amd64" - report_docker_image "$image" "$snyk_project_name" "linux/amd64" - - image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT-arm64" - snyk_project_name="logstash-oss-$version-SNAPSHOT-arm64" - report_docker_image "$image" "$snyk_project_name" "linux/arm64" - - image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT-amd64" - snyk_project_name="logstash-oss-$version-SNAPSHOT-amd64" - report_docker_image "$image" "$snyk_project_name" "linux/amd64" -} - -resolve_version_and_report_docker_images() { - git reset --hard HEAD # reset if any generated files appeared - git checkout "$1" - - # parse version (ex: 8.8.2 from 8.8 branch, or 8.9.0 from main branch) - versions_file="$PWD/versions.yml" - version=$(awk '/logstash:/ { print $2 }' "$versions_file") - report_docker_images "$version" -} - -# resolve docker artifact and report -#for TARGET_BRANCH in "${TARGET_BRANCHES[@]}" -#do -# if git show-ref --quiet refs/heads/"$TARGET_BRANCH"; then -# echo "Using $TARGET_BRANCH branch for docker images." -# resolve_version_and_report_docker_images "$TARGET_BRANCH" -# else -# echo "$TARGET_BRANCH branch doesn't exist." -# fi -#done \ No newline at end of file diff --git a/.buildkite/snyk_report_pipeline.yml b/.buildkite/snyk_report_pipeline.yml index f4306bc1fbe..4f541256ae6 100644 --- a/.buildkite/snyk_report_pipeline.yml +++ b/.buildkite/snyk_report_pipeline.yml @@ -1,12 +1,14 @@ agents: - provider: gcp - imageProject: elastic-images-prod - image: family/platform-ingest-logstash-ubuntu-2204 - machineType: "n2-standard-4" - diskSizeGb: 120 + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" + cpu: "2" + memory: "4Gi" + ephemeralStorage: "64Gi" steps: # reports main, previous (ex: 7.latest) and current (ex: 8.latest) release branches to Snyk - label: ":hammer: Report to Snyk" command: - - .buildkite/scripts/snyk/report.sh \ No newline at end of file + - .buildkite/scripts/snyk/report.sh + retry: + automatic: + - limit: 3 diff --git a/.buildkite/windows_jdk_matrix_pipeline.yml b/.buildkite/windows_jdk_matrix_pipeline.yml index 2c336e1418b..a54e7eca5fb 100644 --- a/.buildkite/windows_jdk_matrix_pipeline.yml +++ b/.buildkite/windows_jdk_matrix_pipeline.yml @@ -33,20 +33,14 @@ steps: value: "adoptiumjdk_21" - label: "Adoptium JDK 17 (Eclipse Temurin)" value: "adoptiumjdk_17" - - label: "Adoptium JDK 11 (Eclipse Temurin)" - value: "adoptiumjdk_11" - label: "OpenJDK 21" value: "openjdk_21" - label: "OpenJDK 17" value: "openjdk_17" - - label: "OpenJDK 11" - value: "openjdk_11" - label: "Zulu 21" value: "zulu_21" - label: "Zulu 17" value: "zulu_17" - - label: "Zulu 11" - value: "zulu_11" - wait: ~ if: build.source != "schedule" && build.source != "trigger_job" diff --git a/.fossa.yml b/.fossa.yml index 55e69801bbc..60fd562b6da 100755 --- a/.fossa.yml +++ b/.fossa.yml @@ -21,10 +21,6 @@ analyze: type: gradle target: 'dependencies-report:' path: . - - name: ingest-converter - type: gradle - target: 'ingest-converter:' - path: . - name: logstash-core type: gradle target: 'logstash-core:' diff --git a/.ruby-version b/.ruby-version index 57fc792f86a..dac625a8779 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -jruby-9.3.10.0 +jruby-9.4.8.0 diff --git a/Gemfile.template b/Gemfile.template index 9989269fb72..66eea71b5f2 100644 --- a/Gemfile.template +++ b/Gemfile.template @@ -14,6 +14,7 @@ gem "logstash-output-elasticsearch", ">= 11.14.0" gem "polyglot", require: false gem "treetop", require: false gem "faraday", "~> 1", :require => false # due elasticsearch-transport (elastic-transport) depending faraday '~> 1' +gem "minitar", :group => :build gem "childprocess", "~> 4", :group => :build gem "fpm", "~> 1", ">= 1.14.1", :group => :build # compound due to bugfix https://github.com/jordansissel/fpm/pull/1856 gem "gems", "~> 1", :group => :build diff --git a/bin/ingest-convert.bat b/bin/ingest-convert.bat deleted file mode 100644 index e9067f42869..00000000000 --- a/bin/ingest-convert.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -cd /d "%~dp0\.." -for /f %%i in ('cd') do set RESULT=%%i - -"%JAVACMD%" -cp "!RESULT!\tools\ingest-converter\build\libs\ingest-converter.jar;*" ^ - org.logstash.ingest.Pipeline %* - -endlocal diff --git a/bin/ingest-convert.sh b/bin/ingest-convert.sh deleted file mode 100755 index 1205cebc5c3..00000000000 --- a/bin/ingest-convert.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -java -cp "$(cd `dirname $0`/..; pwd)"'/tools/ingest-converter/build/libs/ingest-converter.jar:*' \ - org.logstash.ingest.Pipeline "$@" diff --git a/catalog-info.yaml b/catalog-info.yaml index 54a397fd90b..c5367ea6691 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -32,6 +32,7 @@ spec: - resource:logstash-linux-jdk-matrix-pipeline - resource:logstash-windows-jdk-matrix-pipeline - resource:logstash-benchmark-pipeline + - resource:logstash-health-report-tests-pipeline # *********************************** # Declare serverless IT pipeline @@ -613,7 +614,7 @@ spec: kind: Pipeline metadata: name: logstash-benchmark-pipeline - description: ':logstash: The Benchmark pipeline' + description: ':running: The Benchmark pipeline for snapshot version' spec: repository: elastic/logstash pipeline_file: ".buildkite/benchmark_pipeline.yml" @@ -642,4 +643,105 @@ spec: # ******************************* # SECTION END: Benchmark pipeline +# ******************************* + +# *********************************** +# SECTION START: Benchmark Marathon +# *********************************** +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-benchmark-marathon-pipeline + description: Buildkite pipeline for benchmarking multi-version + links: + - title: 'Logstash Benchmark Marathon' + url: https://buildkite.com/elastic/logstash-benchmark-marathon-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: logstash-benchmark-marathon-pipeline + description: ':running: The Benchmark Marathon for multi-version' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/benchmark_marathon_pipeline.yml" + maximum_timeout_in_minutes: 480 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' + SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' + SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' + SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +# ******************************* +# SECTION END: Benchmark Marathon +# ******************************* + +# *********************************** +# Declare Health Report Tests pipeline +# *********************************** +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-health-report-tests-pipeline + description: Buildkite pipeline for the Logstash Health Report Tests + links: + - title: ':logstash Logstash Health Report Tests (Daily, Auto) pipeline' + url: https://buildkite.com/elastic/logstash-health-report-tests-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: logstash-health-report-tests-pipeline + description: ':logstash: Logstash Health Report tests :pipeline:' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/health_report_tests_pipeline.yml" + maximum_timeout_in_minutes: 60 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' + SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' + SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + schedules: + Daily Health Report tests on main branch: + branch: main + cronline: 30 20 * * * + message: Daily trigger of Health Report Tests Pipeline + +# ******************************* +# SECTION END: Health Report Tests pipeline # ******************************* \ No newline at end of file diff --git a/ci/branches.json b/ci/branches.json index dca2ed04dea..470e36cc2c3 100644 --- a/ci/branches.json +++ b/ci/branches.json @@ -5,10 +5,10 @@ "branch": "main" }, { - "branch": "8.15" + "branch": "8.x" }, { - "branch": "8.14" + "branch": "8.15" }, { "branch": "7.17" diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index 5657839d15e..0e63fc37ab0 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -2,12 +2,13 @@ "releases": { "5.x": "5.6.16", "6.x": "6.8.23", - "7.x": "7.17.22", - "8.x": "8.14.3" + "7.x": "7.17.24", + "8.x": "8.15.2" }, "snapshots": { - "7.x": "7.17.23-SNAPSHOT", - "8.x": "8.14.4-SNAPSHOT", - "main": "8.15.0-SNAPSHOT" + "7.x": "7.17.25-SNAPSHOT", + "8.15": "8.15.3-SNAPSHOT", + "8.x": "8.16.0-SNAPSHOT", + "main": "9.0.0-SNAPSHOT" } } diff --git a/config/jvm.options b/config/jvm.options index b729fce9179..f84e904ce57 100644 --- a/config/jvm.options +++ b/config/jvm.options @@ -16,10 +16,6 @@ ## ################################################################ -## GC configuration -11-13:-XX:+UseConcMarkSweepGC -11-13:-XX:CMSInitiatingOccupancyFraction=75 -11-13:-XX:+UseCMSInitiatingOccupancyOnly ## Locale # Set the locale language @@ -59,11 +55,7 @@ #-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof ## GC logging -#-Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,pid,tags:filecount=32,filesize=64m - -# log GC status to a file with time stamps -# ensure the directory exists -#-Xloggc:${LS_GC_LOG_FILE} +#-Xlog:gc*,gc+age=trace,safepoint:file=${LS_GC_LOG_FILE}:utctime,pid,tags:filecount=32,filesize=64m # Entropy source for randomness -Djava.security.egd=file:/dev/urandom diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 34dad2ee375..adbd2ce6a61 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -116,8 +116,6 @@ include::static/managing-multiline-events.asciidoc[] include::static/glob-support.asciidoc[] -include::static/ingest-convert.asciidoc[] - include::static/field-reference.asciidoc[] //The `field-reference.asciidoc` file (included above) contains a diff --git a/docs/static/breaking-changes.asciidoc b/docs/static/breaking-changes.asciidoc index 7a7107869e5..683a76a6749 100644 --- a/docs/static/breaking-changes.asciidoc +++ b/docs/static/breaking-changes.asciidoc @@ -12,7 +12,22 @@ Logstash 8.0.0 and later. NOTE: Migrating directly between non-consecutive major versions (6.x to 8.x) is not recommended. -We suggest that you upgrade to {prev-major-last} before you upgrade to 8.0, +We suggest that you upgrade to {prev-major-last} before you upgrade to 8.0. + +[discrete] +[[breaking-8.15]] +=== 8.15 + +*New {ls} SNMP integration plugin* + +Prior to 8.15.0, {ls} bundled stand-alone versions of the `input-snmp` and `input-snmptrap` plugins by default. +When you upgrade to 8.15.0, the stand-alone plugins are replaced by the 4.0.0+ version contained in the new SNMP integration plugin. + +IMPORTANT: Before you upgrade to {ls} 8.15.0, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in the {logstash-ref}/plugins-integrations-snmp.html[SNMP integration plugin]. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have some options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. + +[discrete] +=== Breaking changes in earlier releases See these topics for breaking changes in earlier releases: diff --git a/docs/static/event-data.asciidoc b/docs/static/event-data.asciidoc index 78713346f56..9952fc168aa 100644 --- a/docs/static/event-data.asciidoc +++ b/docs/static/event-data.asciidoc @@ -149,6 +149,12 @@ filter { } ---------------------------------- +If an expression generates an error when it is evaluated, event processing stops and a warning message is written to the log. +For example, comparing integer value `100` with string value `"100"` cannot be evaluated with certainty, and so processing stops and the error is logged. + +To capture the full content of the message at the time the error occurs, set the log level to `debug`. +Check out <> for more information about how to configure logging and available log levels. + You can specify multiple expressions in a single condition: [source,js] diff --git a/docs/static/filebeat-modules.asciidoc b/docs/static/filebeat-modules.asciidoc index 404e21cf22f..257bc8623a3 100644 --- a/docs/static/filebeat-modules.asciidoc +++ b/docs/static/filebeat-modules.asciidoc @@ -134,10 +134,6 @@ For a full example, see <>. //* <> //* <> // -//TIP: {ls} provides an <> -//to help you migrate ingest pipeline definitions to {ls} configs. The tool does -//not currently support all the processors that are available for ingest node, but -//it's a good starting point. // //[[parsing-apache2]] //==== Apache 2 Logs diff --git a/docs/static/ingest-convert.asciidoc b/docs/static/ingest-convert.asciidoc deleted file mode 100644 index b263faebde4..00000000000 --- a/docs/static/ingest-convert.asciidoc +++ /dev/null @@ -1,90 +0,0 @@ -[[ingest-converter]] -=== Converting Ingest Node Pipelines - -After implementing {ref}/ingest.html[ingest] pipelines to parse your data, you -might decide that you want to take advantage of the richer transformation -capabilities in Logstash. For example, you may need to use Logstash instead of -ingest pipelines if you want to: - -* Ingest from more inputs. Logstash can natively ingest data from many other -sources like TCP, UDP, syslog, and relational databases. - -* Use multiple outputs. Ingest node was designed to only support Elasticsearch -as an output, but you may want to use more than one output. For example, you may -want to archive your incoming data to S3 as well as indexing it in -Elasticsearch. - -* Take advantage of the richer transformation capabilities in Logstash, such as -external lookups. - -* Use the persistent queue feature to handle spikes when ingesting data (from -Beats and other sources). - -To make it easier for you to migrate your configurations, Logstash provides an -ingest pipeline conversion tool. The conversion tool takes the ingest pipeline -definition as input and, when possible, creates the equivalent Logstash -configuration as output. - -See <> for a full list of tool limitations. - -[[ingest-converter-run]] -==== Running the tool - -You'll find the conversion tool in the `bin` directory of your Logstash -installation. See <> to find the location of `bin` on your system. - -To run the conversion tool, use the following command: - -[source,shell] ------ -bin/ingest-convert.sh --input INPUT_FILE_URI --output OUTPUT_FILE_URI [--append-stdio] ------ - -Where: - -* `INPUT_FILE_URI` is a file URI that specifies the full path to the JSON file -that defines the ingest node pipeline. - -* `OUTPUT_FILE_URI` is the file URI of the Logstash DSL file that will be -generated by the tool. - -* `--append-stdio` is an optional flag that adds stdin and stdout sections to -the config instead of adding the default Elasticsearch output. - -This command expects a file URI, so make sure you use forward slashes and -specify the full path to the file. - -For example: - -[source,text] ------ -bin/ingest-convert.sh --input file:///tmp/ingest/apache.json --output file:///tmp/ingest/apache.conf ------ - - -[[ingest-converter-limitations]] -==== Limitations - -* Painless script conversion is not supported. - -* Only a subset of available processors are -<> for conversion. For -processors that are not supported, the tool produces a warning and continues -with a best-effort conversion. - -[[ingest-converter-supported-processors]] -==== Supported Processors - -The following ingest node processors are currently supported for conversion by -the tool: - -* Append -* Convert -* Date -* GeoIP -* Grok -* Gsub -* Json -* Lowercase -* Rename -* Set diff --git a/docs/static/jvm.asciidoc b/docs/static/jvm.asciidoc index 88765d9077f..51d735b55ff 100644 --- a/docs/static/jvm.asciidoc +++ b/docs/static/jvm.asciidoc @@ -4,8 +4,8 @@ {ls} requires one of these versions: -* Java 11 * Java 17 (default). Check out <> for settings info. +* Java 21 Use the http://www.oracle.com/technetwork/java/javase/downloads/index.html[official @@ -17,13 +17,13 @@ for the official word on supported versions across releases. [[bundled-jdk]] .Bundled JDK [NOTE] -===== +===== {ls} offers architecture-specific https://www.elastic.co/downloads/logstash[downloads] that include -Adoptium Eclipse Temurin 17, the latest long term support (LTS) release of the JDK. +Adoptium Eclipse Temurin 17, a long term support (LTS) release of the JDK. Use the LS_JAVA_HOME environment variable if you want to use a JDK other than the -version that is bundled. +version that is bundled. If you have the LS_JAVA_HOME environment variable set to use a custom JDK, Logstash will continue to use the JDK version you have specified, even after you upgrade. ===== @@ -40,9 +40,9 @@ On systems with Java installed, this command produces output similar to the foll [source,shell] ----- -java version "11.0.1" 2018-10-16 LTS -Java(TM) SE Runtime Environment 18.9 (build 11.0.1+13-LTS) -Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.1+13-LTS, mixed mode) +openjdk version "17.0.12" 2024-07-16 +OpenJDK Runtime Environment Temurin-17.0.12+7 (build 17.0.12+7) +OpenJDK 64-Bit Server VM Temurin-17.0.12+7 (build 17.0.12+7, mixed mode) ----- [float] diff --git a/docs/static/monitoring/monitoring-apis.asciidoc b/docs/static/monitoring/monitoring-apis.asciidoc index f6b44e258d1..897507d1e22 100644 --- a/docs/static/monitoring/monitoring-apis.asciidoc +++ b/docs/static/monitoring/monitoring-apis.asciidoc @@ -499,10 +499,6 @@ Example response: "worker_concurrency" : { "current": 1.973, "lifetime": 1.721 - }, - "worker_utilization" : { - "current": 49.32, - "lifetime": 43.02 } } } @@ -540,19 +536,6 @@ Tuning a saturated pipeline to have more workers can often work to increase that A _process_ is also considered "saturated" when its top-level `worker_concurrency` flow metric approaches the _cumulative_ `pipeline.workers` across _all_ pipelines, and similarly can be addressed by tuning the <> that are saturated. -| worker_utilization | - -This is a unitless metric that indicates the percentage of available worker time being used by all plugins in a given pipeline (`duration` / (`uptime` * `pipeline.workers`). -It is useful for determining whether the pipeline has consistently-idle resources or is under resource contention. - -A _pipeline_ is considered "saturated" when its `worker_utilization` flow metric approaches 100, because it indicates that all of its workers are being kept busy. -This is typically an indication of either downstream back-pressure or insufficient resources allocated to the pipeline. -Tuning a saturated pipeline to have more workers can often work to increase that pipeline's throughput and decrease back-pressure to its queue, unless the pipeline is experiencing back-pressure from its outputs. - -A _pipeline_ is considered "starved" when its `worker_utilization` flow metric approaches 0, because it indicates that none of its workers are being kept busy. -This is typically an indication that the inputs are not receiving or retrieving enough volume to keep the pipeline workers busy. -Tuning a starved pipeline to have fewer workers can help it to consume less memory and CPU, freeing up resources for other pipelines. - | queue_backpressure | This is a unitless metric representing the cumulative time spent by all inputs blocked pushing events into their pipeline's queue, relative to wall-clock time (`queue_push_duration_in_millis` / millisecond). It is typically most useful when looking at the stats for an <>. @@ -1002,6 +985,18 @@ A positive number indicates that the queue size-on-disk is growing, and a negati NOTE: The size of a PQ on disk includes both unacknowledged events and previously-acknowledged events from pages that contain one or more unprocessed events. This means it grows gradually as individual events are added, but shrinks in large chunks each time a whole page of processed events is reclaimed (read more: <>). + +| worker_utilization | +This is a unitless metric that indicates the percentage of available worker time being used by this individual plugin (`duration` / (`uptime` * `pipeline.workers`). +It is useful for identifying which plugins in a pipeline are using the available worker resources. + +A _pipeline_ is considered "saturated" when `worker_utilization` approaches 100, because it indicates that all of its workers are being kept busy. +This is typically an indication of either downstream back-pressure or insufficient resources allocated to the pipeline. +Tuning a saturated pipeline to have more workers can often work to increase that pipeline's throughput and decrease back-pressure to its queue, unless the pipeline is experiencing back-pressure from its outputs. + +A _pipeline_ is considered "starved" when `worker_utilization` approaches 0, because it indicates that none of its workers are being kept busy. +This is typically an indication that the inputs are not receiving or retrieving enough volume to keep the pipeline workers busy. +Tuning a starved pipeline to have fewer workers can help it to consume less memory and CPU, freeing up resources for other pipelines. |=== [discrete] diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index 204454a0843..81c1db94cef 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,8 @@ This section summarizes the changes in the following releases: +* <> +* <> * <> * <> * <> @@ -64,6 +66,170 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-15-1]] +=== Logstash 8.15.1 Release Notes + +[[known-issues-8-15-1]] +==== Known issues + +* **{ls} may fail to start under some circumstances.** Single and double quotes are stripped from a pipeline configuration if the configuration includes environment or keystore variable references. +If this situation occurs, {ls} may fail to start or some plugins may use a malformed configuration. +Check out issue https://github.com/elastic/logstash/issues/16437[#16437] for details. ++ +Workaround: Downgrade to {ls} 8.15.0, or temporarily avoid using environment and keystore variable references. + +[[notable-8.15.1]] +==== Performance improvements and notable issues fixed + +* Fixed issue where `${VAR}` incorrectly treated array as a string https://github.com/elastic/logstash/pull/16375[#16375] + +[[core-8.15.1]] +==== Other changes to Logstash core + +* Emit warning message for java version less than 17. Starting from Logstash 9.0, the minimum required version of Java will be Java 17 https://github.com/elastic/logstash/pull/16370[#16370] + +[[plugins-8.15.1]] +==== Plugin releases + +*Elasticsearch Input - 4.20.4* + +* Fixed issue where the `index` parameter was being ignored when using `response_type => aggregations` https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/209[#209] + +*Redis Input - 3.7.1* + +* [Doc] Added `threads` option https://github.com/logstash-plugins/logstash-input-redis/pull/95[#95] + +*Aws Integration - 7.1.8* + +* Added support to all storage classes for s3 output https://github.com/logstash-plugins/logstash-integration-aws/pull/49[#49] + +*Kafka Integration - 11.5.1* + +* [Doc] Added only headers with UTF-8 encoded values are supported in `decorate_events` https://github.com/logstash-plugins/logstash-integration-kafka/pull/174[#174] + + +[[logstash-8-15-0]] +=== Logstash 8.15.0 Release Notes + +[[snmp-ga-8.15.0]] +==== Announcing the new {ls} SNMP integration plugin + +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. + +The `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +[[breaking-8.15.0]] +===== Breaking change with release of `logstash-integration-snmp` + +Prior to 8.15.0, {ls} bundled stand-alone versions of the `input-snmp` and `input-snmptrap` plugins by default. +When you upgrade to 8.15.0, the stand-alone plugins are replaced by the 4.0.0+ version contained in the new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. + +[[featured-8.15.0]] +==== New features and enhancements + +[[wolfi-8.15.0]] +===== Wolfi-flavored Docker Image +Starting with 8.15.0, {ls} provides an extra Docker image flavor based on the Wolfi Linux distribution: https://github.com/wolfi-dev. (https://github.com/elastic/logstash/pull/16189[#16189]). +To use the image append "-wolfi" to the image name: + +``` +docker run docker.elastic.co/logstash/logstash-wolfi:8.15.0 +``` + +This new image flavor builds on top of a smaller and more secure base image, and is planned to become the default flavor in the next major version of {ls}. + +[[notable-8.15.0]] +==== Performance improvements and notable issues fixed +* Improved performance in pipeline-to-pipeline workloads by removing lock contention in event serialization https://github.com/elastic/logstash/pull/16194[#16194] +* Fixed pqcheck handling of paths with spaces on Windows https://github.com/elastic/logstash/pull/16205[#16205] + +[[core-8.15.0]] +==== Other changes to Logstash core + +* Changed rollover strategy in log4j2.properties to delete compressed files older than 7 days https://github.com/elastic/logstash/pull/16179[#16179] +* Fixed handling and reporting of compilation error when a pipeline didn't fit the JVM's ThreadStackSize https://github.com/elastic/logstash/pull/16336[#16336] +* Fixed missing deprecation logging in deprecated Logstash core settings https://github.com/elastic/logstash/pull/16348[#16348] +* Fixed innacurate accounting of `queue_size_in_bytes` metric when using `clean_consumed` option in dead_letter_queue input plugin https://github.com/elastic/logstash/pull/16195[#16195] + +[[dependencies-8.15.0]] +==== Updates to dependencies + +* Updated bundled JDK to 21 https://github.com/elastic/logstash/pull/16055[#16055] +* Updated JRuby to 9.4.8.0 https://github.com/elastic/logstash/pull/16278[#16278] + +[[docs-8.15.0]] +==== Documentation enhancements + +* Logstash {logstash-ref}/monitoring-with-ea.html[monitoring doc] improvements https://github.com/elastic/logstash/pull/16208[#16208] +* Add ecs and datastream requirement for {logstash-ref}/ea-integrations.html#es-tips[integration filter] https://github.com/elastic/logstash/pull/16268[#16268] +* Remove reference to puppet {ls} module https://github.com/elastic/logstash/pull/12356[#12356] +* Add section to describe intended usage of {logstash-ref}/jvm-settings.html#reducing-off-heap-usage[`pipeline.buffer.type`] https://github.com/elastic/logstash/pull/16083[#16083] +* Reposition {logstash-ref}/node-stats-api.html#pipeline-stats[`worker-utilization`] stat for better placement and flow https://github.com/elastic/logstash/pull/16337[#16337] +* Add {logstash-ref}/performance-troubleshooting.html[tuning guidance] based on Flow metrics https://github.com/elastic/logstash/pull/16289[#16289] + + +[[plugins-8.15.0]] +==== Plugin releases + +*De_dot Filter - 1.1.0* + +* Add support for recursively searching sub-fields with the new `recusive =>` config option https://github.com/logstash-plugins/logstash-filter-de_dot/pull/24[#24] + +*Elastic_integration Filter - 0.1.13* + +* Update default elasticsearch tree branch to 8.15 https://github.com/elastic/logstash-filter-elastic_integration/pull/156[#156] + +*Geoip Filter - 7.3.0* + +* Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases https://github.com/logstash-plugins/logstash-filter-geoip/pull/223[#223] +* Updated MaxMind dependencies. + +*Http Filter - 1.6.0* + +* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-filter-http/pull/52[#52] + +*Http_poller Input - 5.6.0* + +* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-input-http_poller/pull/146[#146] + +*Aws Integration - 7.1.7* + +* Remove empty temporary dirs at plugin close https://github.com/logstash-plugins/logstash-integration-aws/pull/46[#46] + +*Kafka Integration - 11.5.0* + +* Add "auto_create_topics" option to allow disabling of topic auto creation https://github.com/logstash-plugins/logstash-integration-kafka/pull/172[#172] + +*Snmp Integration - 4.0.4* + +* Introduce Snmp Integration by default, replacing Snmp Input v1.3.3 and Snmptrap Input v3.1.0 https://github.com/logstash-plugins/logstash-integration-snmp/pull/67[#67] + +*Http Output - 5.7.0* + +* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-output-http/pull/144[#144] + +*Redis Output - 5.2.0* + +* Added support to SSL/TLS configurations https://github.com/logstash-plugins/logstash-output-redis/pull/69[#69] + * `ssl_enabled` + * `ssl_certificate_authorities` + * `ssl_certificate` + * `ssl_key` + * `ssl_verification_mode` + * `ssl_supported_protocols` + * `ssl_cipher_suites` +* Added basic support for SSL https://github.com/logstash-plugins/logstash-output-redis/pull/59[#59] +* Fixed documentation of required settings https://github.com/logstash-plugins/logstash-output-redis/pull/61[#61] + + [[logstash-8-14-3]] === Logstash 8.14.3 Release Notes @@ -237,7 +403,7 @@ No user-facing changes in Logstash core. [[logstash-8-13-1]] === Logstash 8.13.1 Release Notes -No user-facing changes in Logstash core. +* Inject variable reference instead of variable value in `logstash.yaml` to ensure keystore values take precedence over Environment Variables https://github.com/elastic/logstash/pull/16037[#16037] [[logstash-8-13-0]] === Logstash 8.13.0 Release Notes @@ -2318,4 +2484,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437] ==== Plugin releases -Plugins align with release 7.14.0 \ No newline at end of file +Plugins align with release 7.14.0 diff --git a/docs/static/running-logstash-windows.asciidoc b/docs/static/running-logstash-windows.asciidoc index 7a3819c58e6..36a98233e99 100644 --- a/docs/static/running-logstash-windows.asciidoc +++ b/docs/static/running-logstash-windows.asciidoc @@ -46,8 +46,6 @@ Java(TM) SE Runtime Environment 18.9 (build 11.0.3+12-LTS) Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.3+12-LTS, mixed mode) ----- -NOTE: As of the publication of this document, please review this https://github.com/elastic/logstash/issues/10496[known issue that impacts Java 11] before proceeding. - Once you have <> and validated JVM pre-requisites, you may proceed. NOTE: For the examples listed below, we are running Windows Server 2016, Java 11.0.3, diff --git a/logstash-core/lib/logstash/config/mixin.rb b/logstash-core/lib/logstash/config/mixin.rb index deea9717ba2..0d686be3f9c 100644 --- a/logstash-core/lib/logstash/config/mixin.rb +++ b/logstash-core/lib/logstash/config/mixin.rb @@ -93,7 +93,7 @@ def config_init(params) # Resolve environment variables references params.each do |name, value| - params[name.to_s] = deep_replace(value) + params[name.to_s] = deep_replace(value, true) end # Intercept codecs that have not been instantiated diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index 3e1aad5f1f9..9cec566ccf0 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -603,6 +603,11 @@ def init_worker_loop "Worker loop initialization error", default_logging_keys(:error => e.message, :exception => e.class, :stacktrace => e.backtrace.join("\n"))) nil + rescue Java::java.lang.StackOverflowError => se + @logger.error( + "Stack overflow error while compiling Pipeline. Please increase thread stack size using -Xss", + default_logging_keys()) + nil end end diff --git a/logstash-core/lib/logstash/runner.rb b/logstash-core/lib/logstash/runner.rb index 70041378563..bdd66cbfda8 100644 --- a/logstash-core/lib/logstash/runner.rb +++ b/logstash-core/lib/logstash/runner.rb @@ -318,9 +318,9 @@ def execute deprecation_logger.deprecated msg end - if JavaVersion::CURRENT < JavaVersion::JAVA_11 - logger.warn I18n.t("logstash.runner.java.version", - :java_home => java.lang.System.getProperty("java.home")) + if JavaVersion::CURRENT < JavaVersion::JAVA_17 + deprecation_logger.deprecated I18n.t("logstash.runner.java.version_17_minimum", + :java_home => java.lang.System.getProperty("java.home")) end logger.warn I18n.t("logstash.runner.java.home") if ENV["JAVA_HOME"] diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index 4e23e34c3a9..d8c75f220af 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -187,7 +187,7 @@ def reset def from_yaml(yaml_path, file_name = "logstash.yml") settings = read_yaml(::File.join(yaml_path, file_name)) - self.merge(deep_replace(flatten_hash(settings)), true) + self.merge(deep_replace(flatten_hash(settings), true), true) self end @@ -197,6 +197,13 @@ def post_process callback.call(self) end end + + # because we cannot rely on deprecation logger being wired up when the setters + # are initially used, we re-emit setter-related deprecations after all post-processing + # hooks have been activated (and therefore after logging has been configured) + @settings.each_value do |setting| + setting.observe_post_process if setting.respond_to?(:observe_post_process) + end end def on_post_process(&block) @@ -839,9 +846,7 @@ def initialize(canonical_proxy, alias_name) end def set(value) - deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set", - :deprecated_alias => name, - :canonical_name => canonical_proxy.name)) + do_log_setter_deprecation super end @@ -856,6 +861,18 @@ def validate_value # bypass deprecation warning wrapped.validate_value if set? end + + def observe_post_process + do_log_setter_deprecation if set? + end + + private + + def do_log_setter_deprecation + deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set", + :deprecated_alias => name, + :canonical_name => canonical_proxy.name)) + end end ## diff --git a/logstash-core/lib/logstash/util/substitution_variables.rb b/logstash-core/lib/logstash/util/substitution_variables.rb index c8d0ead762c..61b938503e5 100644 --- a/logstash-core/lib/logstash/util/substitution_variables.rb +++ b/logstash-core/lib/logstash/util/substitution_variables.rb @@ -28,19 +28,19 @@ module ::LogStash::Util::SubstitutionVariables SECRET_STORE = ::LogStash::Util::LazySingleton.new { load_secret_store } private_constant :SECRET_STORE - # Recursive method to replace substitution variable references in parameters - def deep_replace(value) + # Recursive method to replace substitution variable references in parameters and refine if required + def deep_replace(value, refine = false) if value.is_a?(Hash) value.each do |valueHashKey, valueHashValue| - value[valueHashKey.to_s] = deep_replace(valueHashValue) + value[valueHashKey.to_s] = deep_replace(valueHashValue, refine) end else if value.is_a?(Array) value.each_with_index do |single_value, i| - value[i] = deep_replace(single_value) + value[i] = deep_replace(single_value, refine) end else - return replace_placeholders(value) + return replace_placeholders(value, refine) end end end @@ -49,14 +49,17 @@ def deep_replace(value) # Process following patterns : ${VAR}, ${VAR:defaultValue} # If value matches the pattern, returns the following precedence : Secret store value, Environment entry value, default value as provided in the pattern # If the value does not match the pattern, the 'value' param returns as-is - def replace_placeholders(value) + # When setting refine to true, substituted value will be cleaned against escaped single/double quotes + # and generates array if resolved substituted value is array string + def replace_placeholders(value, refine) if value.kind_of?(::LogStash::Util::Password) - interpolated = replace_placeholders(value.value) + interpolated = replace_placeholders(value.value, refine) return ::LogStash::Util::Password.new(interpolated) end return value unless value.is_a?(String) - value.gsub(SUBSTITUTION_PLACEHOLDER_REGEX) do |placeholder| + is_placeholder_found = false + placeholder_value = value.gsub(SUBSTITUTION_PLACEHOLDER_REGEX) do |placeholder| # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to # the call, so this should be thread-safe. # @@ -65,6 +68,8 @@ def replace_placeholders(value) default = Regexp.last_match(:default) logger.debug("Replacing `#{placeholder}` with actual value") + is_placeholder_found = true + #check the secret store if it exists secret_store = SECRET_STORE.instance replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(SecretStoreExt.getStoreId(name)) @@ -76,8 +81,39 @@ def replace_placeholders(value) end replacement.to_s end + + # no further action need if substitution didn't happen or refine isn't required + return placeholder_value unless is_placeholder_found && refine + + # ENV ${var} value may carry single quote or escaped double quote + # or single/double quoted entries in array string, needs to be refined + refined_value = strip_enclosing_char(strip_enclosing_char(placeholder_value, "'"), '"') + if refined_value.start_with?('[') && refined_value.end_with?(']') + # remove square brackets, split by comma and cleanup leading/trailing whitespace + refined_array = refined_value[1..-2].split(',').map(&:strip) + refined_array.each_with_index do |str, index| + refined_array[index] = strip_enclosing_char(strip_enclosing_char(str, "'"), '"') + end + refined_array + else + refined_value + end end # def replace_placeholders + private + + # removes removed_char of string_value if string_value is wrapped with removed_char + def strip_enclosing_char(string_value, remove_char) + return string_value unless string_value.is_a?(String) + return string_value if string_value.empty? + + if string_value.start_with?(remove_char) && string_value.end_with?(remove_char) + string_value[1..-2] # Remove the first and last characters + else + string_value + end + end + class << self private diff --git a/logstash-core/locales/en.yml b/logstash-core/locales/en.yml index bd9feedff21..10d7a88227c 100644 --- a/logstash-core/locales/en.yml +++ b/logstash-core/locales/en.yml @@ -440,12 +440,12 @@ en: Running Logstash with the bundled JDK is recommended. The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), you can configure LS_JAVA_HOME to use that version instead. - version: >- - Starting from Logstash 8.0, the minimum required version of Java is Java 11; your Java version from - %{java_home} does not meet this requirement. Please reconfigure your version of Java to one that is supported. + version_17_minimum: >- + Starting from Logstash 9.0, the minimum required version of Java is Java 17; + your Java version from `%{java_home}` does not meet this requirement. Running Logstash with the bundled JDK is recommended. The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. - If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), you can configure LS_JAVA_HOME to use that version instead. + If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), the version you supply with LS_JAVA_HOME must meet the minimum requirements. agent: sighup: >- SIGHUP received. diff --git a/logstash-core/spec/logstash/java_pipeline_spec.rb b/logstash-core/spec/logstash/java_pipeline_spec.rb index 8889b87d7dd..d30872e1430 100644 --- a/logstash-core/spec/logstash/java_pipeline_spec.rb +++ b/logstash-core/spec/logstash/java_pipeline_spec.rb @@ -377,6 +377,145 @@ def flush(options) end end + context "when logical expression in conditional" do + extend PipelineHelpers + + let(:settings) { LogStash::SETTINGS.clone } + + config <<-CONFIG + filter { + if [path][to][value] > 100 { + mutate { add_tag => "hit" } + } else { + mutate { add_tag => "miss" } + } + } + CONFIG + + context "raise an error when it's evaluated, should cancel the event execution and log the error" do + context "when type of evaluation doesn't have same type" do + sample_one( [{ "path" => {"to" => {"value" => "101"}}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/no implicit conversion of nil into Integer/) + end + end + + context "when left and right operands of event condition are not comparable" do + context "comparing a non existing field" do + sample_one( [{ "path" => {"to" => "Rome"}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/:/) + end + end + + context "comparing incompatible types" do + sample_one( [{ "path" => {"to" => {"value" => [101, 102]}}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/Unexpected conditional input combination of.*List.*RubyFixnum/) + end + end + end + + context "when the offending logic expression is used in a nested conditional structure" do + config <<-CONFIG + filter { + if "a" == "a" { + if "b" == "b" { + if [path][to][value] > 100 { + mutate { add_tag => "hit" } + } else { + mutate { add_tag => "miss" } + } + } + } + } + CONFIG + + sample_one( [{ "path" => {"to" => {"value" => "101"}}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/no implicit conversion of nil into Integer/) + end + end + + context "when the offending condition is in the output section" do + before do + LogStash::PLUGIN_REGISTRY.add(:input, "spec_sampler_input", PipelineHelpers::SpecSamplerInput) + LogStash::PLUGIN_REGISTRY.add(:output, "spec_sampler_output", PipelineHelpers::SpecSamplerOutput) + end + + describe "given a pipeline executing an event that would trigger an evaluation error" do + let(:pipeline) do + settings.set_value("queue.drain", true) + LogStash::JavaPipeline.new( + org.logstash.config.ir.PipelineConfig.new( + LogStash::Config::Source::Local, :main, + SourceWithMetadata.new( + "config_string", "config_string", + "input { spec_sampler_input {} }\n output { if [path][to][value] > 100 { spec_sampler_output {} } }" + ), settings + ) + ) + end + let(:event) do + [LogStash::Event.new({ "path" => {"to" => {"value" => "101"}}})] + end + + let(:results) do + PipelineHelpers::SpecSamplerInput.set_event event + pipeline.run + PipelineHelpers::SpecSamplerOutput.seen + end + + after do + pipeline.close + end + + subject {results.length > 1 ? results : results.first} + + it "should raise an error without killing the pipeline" do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/no implicit conversion of nil into Integer/) + end + end + + describe "given a pipeline executing an event with invalid UTF-8 string" do + let(:pipeline) do + settings.set_value("queue.drain", true) + LogStash::JavaPipeline.new( + org.logstash.config.ir.PipelineConfig.new( + LogStash::Config::Source::Local, :main, + SourceWithMetadata.new( + "config_string", "config_string", + "input { spec_sampler_input {} }\n output { if [message] =~ /^(NOSQL|SQL):/ { spec_sampler_output {} } }" + ), settings + ) + ) + end + let(:event) do + [LogStash::Event.new({ "message" => "abrac\xC5adabra"})] + end + + let(:results) do + PipelineHelpers::SpecSamplerInput.set_event event + pipeline.run + PipelineHelpers::SpecSamplerOutput.seen + end + + after do + pipeline.close + end + + subject {results.length > 1 ? results : results.first} + + it "should raise an error without killing the pipeline" do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/invalid byte sequence in UTF-8/) + end + end + end + end + end + context "a crashing worker terminates the pipeline and all inputs and workers" do subject { mock_java_pipeline_from_string(config, pipeline_settings_obj) } let(:config) do @@ -1016,6 +1155,39 @@ def flush(options) it_behaves_like 'it flushes correctly' end end + context "Pipeline created with too many filters" do + # create pipeline with 2000 filters + # 2000 filters is more than a thread stack of size 2MB can handle + let(:config) do + <<-EOS + input { dummy_input {} } + filter { + #{" nil_flushing_filter {}\n" * 2000} + } + output { dummy_output {} } + EOS + end + let(:output) { ::LogStash::Outputs::DummyOutput.new } + + before do + allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(output) + allow(LogStash::Plugin).to receive(:lookup).with("input", "dummy_input").and_return(LogStash::Inputs::DummyBlockingInput) + allow(LogStash::Plugin).to receive(:lookup).with("filter", "nil_flushing_filter").and_return(NilFlushingFilterPeriodic) + allow(LogStash::Plugin).to receive(:lookup).with("output", "dummy_output").and_return(::LogStash::Outputs::DummyOutput) + allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain) + end + + let(:pipeline) { mock_java_pipeline_from_string(config, pipeline_settings_obj) } + + it "informs the user that a stack overflow occurred" do + allow(pipeline.logger).to receive(:error) + + pipeline.start + pipeline.shutdown + + expect(pipeline.logger).to have_received(:error).with(/Stack overflow/, anything).at_least(:once) + end + end context "Periodic Flush that intermittently returns nil" do let(:config) do <<-EOS diff --git a/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb b/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb index 9ddd87a783c..481e387ba03 100644 --- a/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb +++ b/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb @@ -52,6 +52,13 @@ end include_examples '#validate_value success' + + context "#observe_post_process" do + it 'does not emit a deprecation warning' do + expect(LogStash::Settings.deprecation_logger).to_not receive(:deprecated).with(a_string_including(deprecated_setting_name)) + settings.get_setting(deprecated_setting_name).observe_post_process + end + end end context "when only the deprecated alias is set" do @@ -71,6 +78,13 @@ include_examples '#validate_value success' + context "#observe_post_process" do + it 're-emits the deprecation warning' do + expect(LogStash::Settings.deprecation_logger).to receive(:deprecated).with(a_string_including(deprecated_setting_name)) + settings.get_setting(deprecated_setting_name).observe_post_process + end + end + it 'validates deprecated alias' do expect { settings.get_setting(canonical_setting_name).deprecated_alias.validate_value }.to_not raise_error end @@ -107,6 +121,13 @@ end include_examples '#validate_value success' + + context "#observe_post_process" do + it 'does not emit a deprecation warning' do + expect(LogStash::Settings.deprecation_logger).to_not receive(:deprecated).with(a_string_including(deprecated_setting_name)) + settings.get_setting(deprecated_setting_name).observe_post_process + end + end end context "when both the canonical setting and deprecated alias are set" do diff --git a/logstash-core/spec/logstash/settings_spec.rb b/logstash-core/spec/logstash/settings_spec.rb index 241ac2272eb..3e0b425dd46 100644 --- a/logstash-core/spec/logstash/settings_spec.rb +++ b/logstash-core/spec/logstash/settings_spec.rb @@ -144,6 +144,18 @@ it "should preserve original settings" do expect(settings.get("foo")).to eq("bar") end + + context 'when a registered setting responds to `observe_post_process`' do + let(:observe_post_process_setting) do + LogStash::Setting::Boolean.new("this.that", true).tap { |s| allow(s).to receive(:observe_post_process) } + end + subject(:settings) do + described_class.new.tap { |s| s.register(observe_post_process_setting) } + end + it 'it sends `observe_post_process`' do + expect(observe_post_process_setting).to have_received(:observe_post_process) + end + end end context "transient settings" do diff --git a/logstash-core/spec/logstash/util/substitution_variables_spec.rb b/logstash-core/spec/logstash/util/substitution_variables_spec.rb new file mode 100644 index 00000000000..13c5c93c43b --- /dev/null +++ b/logstash-core/spec/logstash/util/substitution_variables_spec.rb @@ -0,0 +1,63 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "spec_helper" +require "logstash/util/substitution_variables" + +describe LogStash::Util::SubstitutionVariables do + + subject { Class.new { extend LogStash::Util::SubstitutionVariables } } + + context "ENV or Keystore ${VAR} with single/double quotes" do + # single or double quotes come from ENV/Keystore ${VAR} value + let(:xpack_monitoring_host) { '"http://node1:9200"' } + let(:xpack_monitoring_hosts) { "'[\"http://node3:9200\", \"http://node4:9200\"]'" } + let(:xpack_management_pipeline_id) { '"*"' } + let(:config_string) { + "'input { + stdin { } + beats { port => 5040 } + } + output { + elasticsearch { + hosts => [\"https://es:9200\"] + user => \"elastic\" + password => 'changeme' + } + }'" + } + + # this happens mostly when running LS with docker + it "stripes out quotes" do + expect(subject.send(:strip_enclosing_char, xpack_monitoring_host, '"')).to eql('http://node1:9200') + expect(subject.send(:strip_enclosing_char, xpack_monitoring_hosts, "'")).to eql('["http://node3:9200", "http://node4:9200"]') + expect(subject.send(:strip_enclosing_char, xpack_management_pipeline_id, '"')).to eql('*') + # make sure we keep the hosts, user and password param enclosed quotes + expect(subject.send(:strip_enclosing_char, config_string, "'")).to eql('input { + stdin { } + beats { port => 5040 } + } + output { + elasticsearch { + hosts => ["https://es:9200"] + user => "elastic" + password => \'changeme\' + } + }') + end + end +end \ No newline at end of file diff --git a/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java b/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java index 7bb7b71a256..e851e0fa899 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java @@ -29,6 +29,7 @@ import org.logstash.config.ir.compiler.AbstractFilterDelegatorExt; import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt; import org.logstash.config.ir.compiler.ComputeStepSyntaxElement; +import org.logstash.config.ir.compiler.ConditionalEvaluationError; import org.logstash.config.ir.compiler.Dataset; import org.logstash.config.ir.compiler.DatasetCompiler; import org.logstash.config.ir.compiler.EventCondition; @@ -39,6 +40,7 @@ import org.logstash.config.ir.graph.PluginVertex; import org.logstash.config.ir.graph.Vertex; import org.logstash.config.ir.imperative.PluginStatement; +import org.logstash.execution.AbstractPipelineExt; import org.logstash.execution.QueueBatch; import org.logstash.ext.JrubyEventExtLibrary.RubyEvent; import org.logstash.plugins.ConfigVariableExpander; @@ -96,20 +98,35 @@ public final class CompiledPipeline { */ private final RubyIntegration.PluginFactory pluginFactory; + /** + * Callback listener to manage failed condition evaluation + * */ + private final AbstractPipelineExt.ConditionalEvaluationListener conditionalErrListener; + + public static final class NoopEvaluationListener implements AbstractPipelineExt.ConditionalEvaluationListener { + + @Override + public void notify(ConditionalEvaluationError err) { + // NO-OP + } + } + public CompiledPipeline( final PipelineIR pipelineIR, final RubyIntegration.PluginFactory pluginFactory) { - this(pipelineIR, pluginFactory, null); + this(pipelineIR, pluginFactory, null, new NoopEvaluationListener()); } public CompiledPipeline( final PipelineIR pipelineIR, final RubyIntegration.PluginFactory pluginFactory, - final SecretStore secretStore) + final SecretStore secretStore, + final AbstractPipelineExt.ConditionalEvaluationListener conditionalErrListener) { this.pipelineIR = pipelineIR; this.pluginFactory = pluginFactory; + this.conditionalErrListener = conditionalErrListener; try (ConfigVariableExpander cve = new ConfigVariableExpander( secretStore, EnvironmentVariableProvider.defaultProvider())) { @@ -488,7 +505,7 @@ private SplitDataset split( // by requiring its else branch. if (conditional == null) { final ComputeStepSyntaxElement prepared = - DatasetCompiler.splitDataset(dependencies, condition); + DatasetCompiler.splitDataset(dependencies, condition, conditionalErrListener); conditional = prepared.instantiate(); iffs.put(vertexId, conditional); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java index b34131bcd31..b4f5cf5e041 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java @@ -43,6 +43,10 @@ public ValueSyntaxElement add(final Object obj) { return addField(FieldDefinition.fromValue(definitions.size(), obj)); } + public ValueSyntaxElement add(final String fieldName, final Object obj) { + return addField(FieldDefinition.fromValue(fieldName, obj)); + } + /** * Adds a mutable field of the given type, that doesn't have a default value and is not * initialized by a constructor assignment. @@ -54,6 +58,10 @@ public ValueSyntaxElement add(final Class type) { return addField(FieldDefinition.mutableUnassigned(definitions.size(), type)); } + public ValueSyntaxElement add(final String fieldName, final Class type) { + return addField(FieldDefinition.mutableUnassigned(fieldName, type)); + } + /** * Add a {@link Closure} that should be executed in the constructor after field assignments * have been executed. diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java index 42541f2a56a..eb7e131616b 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java @@ -207,7 +207,7 @@ private static Path debugDir() { } /** - * @return Array of constructor arguments + * @return Map of constructor arguments, key is field's name, value is the instance of the parameter to pass. */ private Map ctorArguments() { final Map result = new HashMap<>(); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java new file mode 100644 index 00000000000..0dd464fc255 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java @@ -0,0 +1,23 @@ +package org.logstash.config.ir.compiler; + +import org.logstash.Event; + +/** + * Exception raised when an if-condition in a pipeline throws an error at runtime. + * */ +public class ConditionalEvaluationError extends RuntimeException { + private static final long serialVersionUID = -8633589068902565868L; + + // This class is serializable because of inheritance from Throwable, however it's not expected + // to be ever transmitted on wire on stored in some binary storage. + private final transient Event failedEvent; + + ConditionalEvaluationError(Throwable cause, Event failedEvent) { + super(cause); + this.failedEvent = failedEvent; + } + + public Event failedEvent() { + return failedEvent; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java index 689fb00302d..956bbe18f1b 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; import org.jruby.RubyArray; import org.jruby.RubyHash; @@ -31,6 +32,7 @@ import org.jruby.runtime.Block; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; +import org.logstash.execution.AbstractPipelineExt; import org.logstash.ext.JrubyEventExtLibrary; /** @@ -55,11 +57,12 @@ private DatasetCompiler() { public static ComputeStepSyntaxElement splitDataset( final Collection parents, - final EventCondition condition) + final EventCondition condition, + final AbstractPipelineExt.ConditionalEvaluationListener conditionalErrListener) { final ClassFields fields = new ClassFields(); - final ValueSyntaxElement ifData = fields.add(new ArrayList<>()); - final ValueSyntaxElement elseData = fields.add(new ArrayList<>()); + final ValueSyntaxElement ifData = fields.add("ifData", new ArrayList<>()); + final ValueSyntaxElement elseData = fields.add("elseData", new ArrayList<>()); final ValueSyntaxElement right = fields.add(DatasetCompiler.Complement.class); final VariableDefinition event = new VariableDefinition(JrubyEventExtLibrary.RubyEvent.class, "event"); @@ -75,23 +78,38 @@ public static ComputeStepSyntaxElement splitDataset( ) ) ); - final ValueSyntaxElement conditionField = fields.add(condition); + final ValueSyntaxElement conditionField = fields.add("condition", condition); final DatasetCompiler.ComputeAndClear compute; + + final ValueSyntaxElement errorNotifier = fields.add(conditionalErrListener); + Closure exceptionHandlerBlock = Closure.wrap( + new SyntaxFactory.MethodCallReturnValue(SyntaxFactory.value("this"), "setDone"), + errorNotifier.call("notify", SyntaxFactory.value("ex")), + SyntaxFactory.ret(ifData)); + ValueSyntaxElement exception = SyntaxFactory.value("ex"); + if (parents.isEmpty()) { + MethodLevelSyntaxElement safeLoop = SyntaxFactory.tryBlock( + conditionalLoop(event, BATCH_ARG, conditionField, ifData, elseData), + ConditionalEvaluationError.class, exceptionHandlerBlock, exception + ); compute = withOutputBuffering( - conditionalLoop(event, BATCH_ARG, conditionField, ifData, elseData), + Closure.wrap(safeLoop), Closure.wrap(clear(elseData)), ifData, fields ); } else { final Collection parentFields = parents.stream().map(fields::add).collect(Collectors.toList()); - final ValueSyntaxElement inputBuffer = fields.add(new ArrayList<>()); + final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", new ArrayList<>()); + MethodLevelSyntaxElement safeLoop = SyntaxFactory.tryBlock( + conditionalLoop(event, inputBuffer, conditionField, ifData, elseData), + ConditionalEvaluationError.class, exceptionHandlerBlock, exception); compute = withOutputBuffering( withInputBuffering( - conditionalLoop(event, inputBuffer, conditionField, ifData, elseData), + Closure.wrap(safeLoop), parentFields, inputBuffer ), - clearSyntax(parentFields).add(clear(elseData)), ifData, fields + clearSyntax(parentFields).add(clear(elseData)).add(clear(inputBuffer)), ifData, fields ); } return ComputeStepSyntaxElement.create( @@ -111,20 +129,18 @@ public static ComputeStepSyntaxElement filterDataset( final AbstractFilterDelegatorExt plugin) { final ClassFields fields = new ClassFields(); - final ValueSyntaxElement outputBuffer = fields.add(new ArrayList<>()); + final ValueSyntaxElement outputBuffer = fields.add("outputBuffer", new ArrayList<>()); final Closure clear = Closure.wrap(); final Closure compute; if (parents.isEmpty()) { compute = filterBody(outputBuffer, BATCH_ARG, fields, plugin); } else { - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList() - ); + final Collection parentFields = createParentStatementsFields(parents, fields); + @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); clear.add(clearSyntax(parentFields)); - final ValueSyntaxElement inputBufferField = fields.add(inputBuffer); + final ValueSyntaxElement inputBufferField = fields.add("inputBuffer", inputBuffer); + compute = withInputBuffering( filterBody(outputBuffer, inputBufferField, fields, plugin), parentFields, inputBufferField @@ -133,6 +149,25 @@ public static ComputeStepSyntaxElement filterDataset( return prepare(withOutputBuffering(compute, clear, outputBuffer, fields)); } + private static Collection createParentStatementsFields(Collection parents, ClassFields fields) { + if (parents.size() == 1) { + return List.of(fields.add("parentStatement", parents.iterator().next())); + } + + final Collection parentFields = new ArrayList<>(); + int i = 0; + for (Dataset parent : parents) { + ValueSyntaxElement add = fields.add("parentStatement" + i, parent); + parentFields.add(add); + i++; + } + return parentFields; + } + + private static boolean isContainedUnderIfStatement(Collection parents) { + return parents.size() == 1 && parents.iterator().next() instanceof SplitDataset; + } + /** *

Builds a terminal {@link Dataset} for the filters from the given parent {@link Dataset}s.

*

If the given set of parent {@link Dataset} is empty the sum is defined as the @@ -155,10 +190,7 @@ public static Dataset terminalFilterDataset(final Collection parents) { } final ClassFields fields = new ClassFields(); - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); final ValueSyntaxElement inputBufferField = fields.add(inputBuffer); final ValueSyntaxElement outputBufferField = fields.add(new ArrayList<>()); @@ -199,10 +231,7 @@ public static Dataset terminalOutputDataset(final Collection parents) { } final ClassFields fields = new ClassFields(); - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); final Closure compute = Closure.wrap(parentFields .stream() .map(DatasetCompiler::computeDataset) @@ -236,7 +265,7 @@ public static ComputeStepSyntaxElement outputDataset( final ClassFields fields = new ClassFields(); final Closure clearSyntax; final Closure computeSyntax; - final ValueSyntaxElement outputField = fields.add(output); + final ValueSyntaxElement outputField = fields.add("outputDelegator", output); if (parents.isEmpty()) { clearSyntax = Closure.EMPTY; computeSyntax = Closure.wrap( @@ -244,8 +273,7 @@ public static ComputeStepSyntaxElement outputDataset( invokeOutput(outputField, BATCH_ARG), unsetPluginIdForLog4j()); } else { - final Collection parentFields = - parents.stream().map(fields::add).collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); @SuppressWarnings("rawtypes") final RubyArray buffer = RubyUtil.RUBY.newArray(); final Closure inlineClear; @@ -256,7 +284,7 @@ public static ComputeStepSyntaxElement outputDataset( inlineClear = Closure.EMPTY; clearSyntax = clearSyntax(parentFields); } - final ValueSyntaxElement inputBuffer = fields.add(buffer); + final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", buffer); computeSyntax = withInputBuffering( Closure.wrap( setPluginIdForLog4j(outputField), @@ -283,7 +311,7 @@ private static Closure filterBody( final ClassFields fields, final AbstractFilterDelegatorExt plugin) { - final ValueSyntaxElement filterField = fields.add(plugin); + final ValueSyntaxElement filterField = fields.add("plugin", plugin); final Closure body = Closure.wrap( setPluginIdForLog4j(filterField), buffer(outputBuffer, filterField.call("multiFilter", inputBuffer)) @@ -334,11 +362,15 @@ private static ComputeStepSyntaxElement prepare(final DatasetCompiler.C */ private static Closure withInputBuffering(final Closure compute, final Collection parents, final ValueSyntaxElement inputBuffer) { - return Closure.wrap( - parents.stream().map(par -> SyntaxFactory.value("org.logstash.config.ir.compiler.Utils") - .call("copyNonCancelledEvents", computeDataset(par), inputBuffer) - ).toArray(MethodLevelSyntaxElement[]::new) - ).add(compute).add(clear(inputBuffer)); + return computeNonCancelledEventsBlock(parents, inputBuffer) + .add(compute) + .add(clear(inputBuffer)); + } + + private static Closure computeNonCancelledEventsBlock(Collection parents, ValueSyntaxElement inputBuffer) { + return Closure.wrap(parents.stream().map(par -> SyntaxFactory.value("org.logstash.config.ir.compiler.Utils") + .call("copyNonCancelledEvents", computeDataset(par), inputBuffer) + ).toArray(MethodLevelSyntaxElement[]::new)); } /** @@ -378,14 +410,14 @@ private static MethodLevelSyntaxElement callFilterFlush(final ClassFields fields final boolean shutdownOnly) { final MethodLevelSyntaxElement condition; final ValueSyntaxElement flushArgs; - final ValueSyntaxElement flushFinal = fields.add(flushOpts(true)); + final ValueSyntaxElement flushFinal = fields.add("shutdownFlushOptions", flushOpts(true)); if (shutdownOnly) { condition = SyntaxFactory.and(FLUSH_ARG, SHUTDOWN_ARG); flushArgs = flushFinal; } else { condition = FLUSH_ARG; flushArgs = SyntaxFactory.ternary( - SHUTDOWN_ARG, flushFinal, fields.add(flushOpts(false)) + SHUTDOWN_ARG, flushFinal, fields.add("flushOptions", flushOpts(false)) ); } return SyntaxFactory.ifCondition( @@ -490,9 +522,13 @@ public Collection compute(@SuppressWarnings("raw if (done) { return data; } - parent.compute(batch, flush, shutdown); - done = true; - return data; + try { + parent.compute(batch, flush, shutdown); + done = true; + return data; + } catch (ConditionalEvaluationError ex) { + return data; + } } @Override diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java index a68ec8a9888..62e3a3e69b5 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java @@ -685,19 +685,19 @@ private static final class UnexpectedTypeException extends IllegalArgumentExcept UnexpectedTypeException(final Expression left, final Expression right) { super( String.format( - "Unexpected input types left: %s, right: %s", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) + "Unexpected conditional input combination of '%s' (left) and '%s' (right)", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) ) ); } UnexpectedTypeException(final Object inner) { - super(String.format("Unexpected input type %s", getUnexpectedTypeDetails(inner))); + super(String.format("Unexpected conditional input type '%s'", getUnexpectedTypeDetails(inner))); } UnexpectedTypeException(final Object left, final Object right) { super( String.format( - "Unexpected input type combination left %s, right %s", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) + "Unexpected conditional input combination of '%s' (left) and '%s' (right)", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) ) ); } @@ -709,9 +709,13 @@ private static String getUnexpectedTypeDetails(Object unexpected) { details = (expression.getSourceWithMetadata() != null) ? expression.getSourceWithMetadata().toString() : expression.toString(); } else { - details = unexpected.toString(); + if (unexpected == null) { + details = ""; + } else { + details = unexpected.toString(); + } } - return String.format("%s:%s", unexpected.getClass(), details); + return String.format("%s:%s", unexpected == null ? "" : unexpected.getClass(), details); } } } diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java index b3e481e6fd4..4ed2ba3b20e 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java @@ -46,6 +46,12 @@ public static FieldDefinition fromValue(final int index, final Object value) { ); } + public static FieldDefinition fromValue(final String fieldName, final Object value) { + return new FieldDefinition( + variableDefinition(value.getClass(), fieldName), false, null, value + ); + } + /** * Creates a mutable field with given type and without an assigned value. * @param index Index for naming @@ -58,6 +64,12 @@ public static FieldDefinition mutableUnassigned(final int index, final Class ); } + public static FieldDefinition mutableUnassigned(final String fieldName, final Class type) { + return new FieldDefinition( + variableDefinition(type, fieldName), true, null, null + ); + } + private FieldDefinition(final VariableDefinition typeDef, final boolean mutable, final SyntaxElement initializer, final Object ctorArgument) { this.def = typeDef; @@ -98,4 +110,9 @@ public String generateCode() { private static VariableDefinition variableDefinition(final Class type, final int index) { return new VariableDefinition(type, String.format("field%d", index)); } + + private static VariableDefinition variableDefinition(final Class type, final String fieldName) { + return new VariableDefinition(type, String.format("%sField", fieldName)); + } + } diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java index f4076560283..c8134719470 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java @@ -88,6 +88,14 @@ public static MethodLevelSyntaxElement forLoop(final VariableDefinition element, ); } + public static MethodLevelSyntaxElement tryBlock(final Closure body, final Class exceptionToCatch, Closure exceptionHandlerBlock, + ValueSyntaxElement exception) { + return () -> SyntaxFactory.join( + "try {\n", body.generateCode(), + "\n} catch (" + exceptionToCatch.getCanonicalName() + " " + exception.generateCode() + ") { " + exceptionHandlerBlock.generateCode() + " }" + ); + } + public static MethodLevelSyntaxElement ifCondition(final MethodLevelSyntaxElement condition, final Closure body) { return ifCondition(condition, body, Closure.EMPTY); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java index 3bf64beb058..cb6341058d8 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java @@ -20,6 +20,8 @@ package org.logstash.config.ir.compiler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.logstash.ext.JrubyEventExtLibrary; import java.util.Collection; @@ -29,6 +31,7 @@ * Static utility methods that replace common blocks of generated code in the Java execution. */ public class Utils { + private static final Logger logger = LogManager.getLogger(Utils.class); @SuppressWarnings({"unchecked", "rawtypes"}) // has field1.compute(batchArg, flushArg, shutdownArg) passed as input @@ -44,7 +47,15 @@ public static void copyNonCancelledEvents(Collection input, EventCondition filter, List fulfilled, List unfulfilled) { for (JrubyEventExtLibrary.RubyEvent e : input) { - if (filter.fulfilled(e)) { + boolean isFulfilled; + try { + isFulfilled = filter.fulfilled(e); + } catch (org.jruby.exceptions.TypeError | IllegalArgumentException | org.jruby.exceptions.ArgumentError ex) { + // in case of error evaluation of an if condition, cancel the event + e.getEvent().cancel(); + throw new ConditionalEvaluationError(ex, e.getEvent()); + } + if (isFulfilled) { fulfilled.add(e); } else { unfulfilled.add(e); diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index 82cd216e64e..0ac584f1595 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -21,6 +21,8 @@ package org.logstash.execution; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; @@ -76,6 +78,7 @@ import org.logstash.config.ir.PipelineIR; import org.logstash.config.ir.compiler.AbstractFilterDelegatorExt; import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt; +import org.logstash.config.ir.compiler.ConditionalEvaluationError; import org.logstash.execution.queue.QueueWriter; import org.logstash.ext.JRubyAbstractQueueWriteClientExt; import org.logstash.ext.JRubyWrappedWriteClientExt; @@ -163,10 +166,33 @@ public class AbstractPipelineExt extends RubyBasicObject { private @SuppressWarnings("rawtypes") RubyArray filters; private @SuppressWarnings("rawtypes") RubyArray outputs; + private String lastErrorEvaluationReceived = ""; + public AbstractPipelineExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + public interface ConditionalEvaluationListener { + void notify(ConditionalEvaluationError err); + } + + public final class LogErrorEvaluationListener implements ConditionalEvaluationListener { + @Override + public void notify(ConditionalEvaluationError err) { + lastErrorEvaluationReceived = err.getCause().getMessage(); + LOGGER.warn("{}. Event was dropped, enable debug logging to see the event's payload.", lastErrorEvaluationReceived); + LOGGER.debug("Event generating the fault: {}", err.failedEvent().toMap().toString()); + + // logs the exception at debug level + try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { + err.printStackTrace(pw); + LOGGER.debug("{}", sw); + } catch (IOException ioex) { + LOGGER.warn("Invalid operation on closing internal resources", ioex); + } + } + } + @JRubyMethod(required = 4) public AbstractPipelineExt initialize(final ThreadContext context, final IRubyObject[] args) throws IncompleteSourceWithMetadataException, NoSuchAlgorithmException { @@ -183,7 +209,8 @@ public AbstractPipelineExt initialize(final ThreadContext context, final IRubyOb ).initialize(context, args[3], this, dlqWriter(context)), RubyUtil.FILTER_DELEGATOR_CLASS ), - getSecretStore(context) + getSecretStore(context), + new LogErrorEvaluationListener() ); inputs = RubyArray.newArray(context.runtime, lirExecution.inputs()); filters = RubyArray.newArray(context.runtime, lirExecution.filters()); @@ -828,4 +855,10 @@ public IRubyObject isShutdownRequested(final ThreadContext context) { // shutdown_requested? MUST be implemented in the concrete implementation of this class. throw new IllegalStateException("Pipeline implementation does not provide `shutdown_requested?`, which is a Logstash internal error."); } + + @VisibleForTesting + @JRubyMethod(name = "last_error_evaluation_received") + public final RubyString getLastErrorEvaluationReceived(final ThreadContext context) { + return RubyString.newString(context.runtime, lastErrorEvaluationReceived); + } } diff --git a/logstash-core/src/main/java/org/logstash/util/JavaVersion.java b/logstash-core/src/main/java/org/logstash/util/JavaVersion.java index 39eee187d02..c1d619e9857 100644 --- a/logstash-core/src/main/java/org/logstash/util/JavaVersion.java +++ b/logstash-core/src/main/java/org/logstash/util/JavaVersion.java @@ -30,7 +30,8 @@ public class JavaVersion implements Comparable { public static final JavaVersion CURRENT = parse(System.getProperty("java.specification.version")); - public static final JavaVersion JAVA_11 = parse("11"); + public static final JavaVersion JAVA_17 = parse("17"); + private final List version; private JavaVersion(List version){ diff --git a/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java b/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java index 4881035a090..c38f4e29e81 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java @@ -26,6 +26,7 @@ import org.logstash.Event; import org.logstash.FieldReference; import org.logstash.RubyUtil; +import org.logstash.config.ir.CompiledPipeline; import org.logstash.config.ir.PipelineTestUtil; import org.logstash.ext.JrubyEventExtLibrary; @@ -54,7 +55,8 @@ public void compilesOutputDataset() { public void compilesSplitDataset() { final FieldReference key = FieldReference.from("foo"); final SplitDataset left = DatasetCompiler.splitDataset( - Collections.emptyList(), event -> event.getEvent().includes(key) + Collections.emptyList(), event -> event.getEvent().includes(key), + new CompiledPipeline.NoopEvaluationListener() ).instantiate(); final Event trueEvent = new Event(); trueEvent.setField(key, "val"); diff --git a/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf b/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf index 4aa09f7dcf8..a612588e9dd 100644 --- a/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf +++ b/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf @@ -17,6 +17,7 @@ filter { if [a] { if [a] { if [a] { + noop {} noop {} if [a] { noop {} } } @@ -25,7 +26,8 @@ filter { } output { if [a] { noop {} } + else { noop {} } stdout {} stdout {} - stdout {} +# stdout {} } diff --git a/qa/docker/shared_examples/container_options.rb b/qa/docker/shared_examples/container_options.rb index 1d5108bff94..48c51d382b9 100644 --- a/qa/docker/shared_examples/container_options.rb +++ b/qa/docker/shared_examples/container_options.rb @@ -56,4 +56,40 @@ expect(get_settings(@container)['pipeline.unsafe_shutdown']).to be_truthy end end + + context 'when setting config.string' do + let(:options) { + { + 'ENV' => [ + 'USER=kimchy', + 'CONFIG_STRING=input { + beats { port => 5040 } + } + output { + elasticsearch { + hosts => ["https://es:9200"] + user => "${USER}" + password => \'changeme\' + } + }' + ] + } + } + + it "persists ${CONFIG_STRING} key in logstash.yml, resolves when running and spins up without issue" do + settings = get_settings(@container) + expect(settings['config.string']).to eq("${CONFIG_STRING}") + + pipeline_config = get_pipeline_stats(@container) + input_plugins = pipeline_config.dig('plugins', 'inputs') + expect(input_plugins[0].dig('name')).to eql('beats') + + output_plugins = pipeline_config.dig('plugins', 'outputs') + expect(output_plugins[0].dig('name')).to eql('elasticsearch') + + # check if logs contain the ES request with the resolved ${USER} + container_logs = @container.logs(stdout: true) + expect(container_logs.include?('https://kimchy:xxxxxx@es:9200')).to be true + end + end end diff --git a/qa/docker/shared_examples/xpack.rb b/qa/docker/shared_examples/xpack.rb index c1706a69fc0..254e0354ca9 100644 --- a/qa/docker/shared_examples/xpack.rb +++ b/qa/docker/shared_examples/xpack.rb @@ -2,21 +2,61 @@ before do @image = find_image(flavor) - @container = start_container(@image, {'ENV' => env}) end after do cleanup_container(@container) end - context 'when configuring xpack settings' do - let(:env) { %w(xpack.monitoring.enabled=false xpack.monitoring.elasticsearch.hosts=["http://node1:9200","http://node2:9200"]) } + describe 'when configuring xpack settings' do - it 'persists monitoring environment var keys' do - # persisting actual value of the environment keys bring the issue where keystore looses its power - # visit https://github.com/elastic/logstash/issues/15766 for details - expect(get_settings(@container)['xpack.monitoring.enabled']).to eq("${xpack.monitoring.enabled}") - expect(get_settings(@container)['xpack.monitoring.elasticsearch.hosts']).to eq("${xpack.monitoring.elasticsearch.hosts}") + context 'when persists env var keys into logstash.yml' do + let(:env) { %w(XPACK_MONITORING_ENABLED=false XPACK_MONITORING_ELASTICSEARCH_HOSTS=["http://node1:9200","http://node2:9200"]) } + + before do + @container = start_container(@image, {'ENV' => env}) + end + + it 'saves keys instead actual value which will be resolved from keystore | env later' do + settings = get_settings(@container) + expect(settings['xpack.monitoring.enabled']).to eq("${XPACK_MONITORING_ENABLED}") + expect(settings['xpack.monitoring.elasticsearch.hosts']).to eq("${XPACK_MONITORING_ELASTICSEARCH_HOSTS}") + end + end + + context 'with running with env vars' do + let(:env) { + [ + 'XPACK_MONITORING_ENABLED=true', + 'XPACK_MONITORING_ELASTICSEARCH_HOSTS="http://node1:9200"', + 'XPACK_MANAGEMENT_ENABLED=true', + 'XPACK_MANAGEMENT_PIPELINE_ID=["*"]', # double quotes intentionally placed + 'XPACK_MANAGEMENT_ELASTICSEARCH_HOSTS=["http://node3:9200", "http://node4:9200"]' + ] + } + + it 'persists var keys into logstash.yml and uses their resolved actual values' do + container = create_container(@image, {'ENV' => env}) + + sleep(15) # wait for container run + + settings = get_settings(container) + + expect(settings['xpack.monitoring.enabled']).to eq("${XPACK_MONITORING_ENABLED}") + expect(settings['xpack.monitoring.elasticsearch.hosts']).to eq("${XPACK_MONITORING_ELASTICSEARCH_HOSTS}") + expect(settings['xpack.management.enabled']).to eq("${XPACK_MANAGEMENT_ENABLED}") + expect(settings['xpack.management.pipeline.id']).to eq("${XPACK_MANAGEMENT_PIPELINE_ID}") + expect(settings['xpack.management.elasticsearch.hosts']).to eq("${XPACK_MANAGEMENT_ELASTICSEARCH_HOSTS}") + + # get container logs + container_logs = container.logs(stdout: true) + + # check if logs contain node3 & node4 values actually resolved and used + expect(container_logs.include?('pipeline_id=>["*"]')).to be true + # note that, we are not spinning up ES nodes, so values can be in errors or in pool update logs + expect(container_logs.include?('http://node3:9200')).to be true + expect(container_logs.include?('http://node4:9200')).to be true + end end end end diff --git a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt index e772b2ea1fa..ca9ed7cba5a 100644 --- a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt +++ b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzI4WhcNMjQwNzE1MTgyNzI4WjBVMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTAzWhcNMjcwNDEyMTUzNTAzWjBVMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxGDAWBgNVBAMMD2ludGVybWVkaWF0ZS1jYTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAPHXRKHl/LsaDFGnys0b4mlLlGQ6qJSC5ut8FZoa -B7kDeHsi8TiFVg/RENghFBClP9aGhWxndGSeQTDkg+KPxKovXBX1J9mP+z+ol47V -QVeCT++AvPQYmtLNLwZg0e5GVHFMnVq64LlTc1XjvzjlU751A5dEq/miMpovx7fS -oI2zxtVOYx+BF4hr+WA20mxbQoGXVM8ITDBWySM5IDhngKNAfmXNQZFffJgk+pwe -++I4KzRkLU02OEUnH63/YFkn+CoZBmg+II8cNItRVYCRifXb8tsO6OEwQQEe1RSk -fmxiiiTYCDMB5GUSowCRotKTwEXgWwTLKghzzUoCPJapsb/CzY2xp/ECtQt420CJ -aosx/fLmcuKmbhW847hYq4L8HqZvjs3i7E6wjuUAgn/um28kU1PrN9l6qTJhHKKP -GZUCC/Cou0g9q7Y9ad672mWgExo0I1vXm2H1KNYy94agmXaIgmelvPBTPREkEEL6 -MnfrZKiYOG5wjSMubg0L4JYKUMuToXvfXxe6o2rDofe/oTUXErGDYV4z7vr58dQg -TM5PtxpZCLmqELaBFydB5hEBQly8Fxh3SVpuwkcaCxOlXMkdvHEK9daYfE9jV0fu -StfckkCLdDZMHI+iSdQsqK6E9vc4tX2rnhzgKex0Byu0GUEg6XoFL2GKdUOvRZTJ -/tMxAgMBAAGjZjBkMB0GA1UdDgQWBBR6wQ0Is6YucsLFNwdZ3ANX1jrgsjAfBgNV -HSMEGDAWgBSWO1UVkFl5LOuplY08xBLcfGyc+jASBgNVHRMBAf8ECDAGAQH/AgEA -MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQUFAAOCAgEANoo45TDx1lfglAmJ -8nt/58y7fm8trxmrSQmDAIuXZKqATNsEw/+VbIXqeC9sT2lWY9URFmmFm505JP+1 -43nM68cjfAK7ihLcGS7XwQMtQCgiQzJKeZFOx/l30lN5rRje5HHbSIF+uuwO987J -LNYJ+rS7N25lB5EQY8satuZWMVtSZ7YQCMkSJve7KjfrbE3kORjHs6jn3bdyuWAr -oApgKJgCUX3JSPZyH3T/xakMneZNkYr0CC1kNi9wL4js5I/nN2Z4er0N1Zt6PIM+ -hIJcW/Qv0BtI4bzkNkGSehm+Ofr+njuqWaaQMF7kLPTGDg/0SN63Y+X59CJQMnjF -+oW0oGm30S3okFpgXSU+DDxNPlh1j3jTMoArPvIaTceImfmsBXU62JRXQWaAr1jF -EujYbTJ147/7NmsmnskFO37Mp35JxL3ELWfMeBAL/HC11mzUbGFhtvtFqhQMWMbt -5AAR9ZUaPO8xTX6Ag8/8MXF5wFRkJBqTNn6GaX5PUrWhlAKpGuAuuR1NBY5TGUHX -NcFu+bf5JtjhgSUGB0FuZfdZvw46k30NLlT39NbXayY102XYfrAsl0X0TNqjQua1 -7j7FqYrAALQVLfYeKrE0nKaH35nU6Kk1Iy42CbB71MHqcg7DTjn0MvTKZcnQ8pxv -RQwP+BXkzwfWWEp05rlVeKS0IjQ= +AQEBBQADggIPADCCAgoCggIBALUBL8nGcBsOrfv/uI555eOiBgFYD+V3etaiFDi6 +MtZynj9oazrssevjNwP045j8a2L9apjeq9U5REs2MrYoHTEPoIeOiFPNQfiBtJt/ +DjBiXYQbrSfpkjPmRREPKkTX9lFNSlEmG3y+kNMyASvR3K4IM6Bqd+FYt/GUyhZP +LFx9ZhRAiq5uulLqVaAdAp+wdjAXeSedtCTs+20aW+NKA32+Pb8vzJp6HOmh+ZXl +orlEyKKfUYI3ODu9IoXTu6ZN8XZ4LfrDu47wswgQU4n2TOn+Qm7nwE9GIjQBjYOz +s3eUYyzE/z5oyrC6t2Z2ZEq2pCwMwf7iPXRFrgSvgLJCCSdwrB/bTxghM0vmFeoV +V2vfTp3VC/WbSW59tV++iMyMQTPw+grg0NEJCrwNnFJ1DL9iT0f902z2mknxuCM+ +2HGpt2EG+FnU4kR2gHQKt4BNGIuscchp8J85CGjheoNvuHaUktVWwqUaHUL8b9MV +dav10Obfr0+bWNIY2xsufM8sX/QP3VsfJ/nFkoImYNW5SI5g7/5DTIxIE/z8Ab1a +4JT2JCzPO8wDFyle4RVOG+2WfDBUkkjIjXlfxBwLneTPzqv5jmpPIwcpdF0c9bWw +Jwb/e4hB/bAFwZvLFr0aypJ0Zqbc/67n5HDsKx8+/XUKar1qPP6AzrfecOmBwwOH +AsE7AgMBAAGjZjBkMB0GA1UdDgQWBBT5X+u4LkGBED9seTSNYgA40tXJUTAfBgNV +HSMEGDAWgBQB2VzO86glAf1wb5+bjW8YPxZlpTASBgNVHRMBAf8ECDAGAQH/AgEA +MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAGb6iGwxKXTSgxP96 +4UMtpKYFGykBuajxr105jFSjXArhjj5/U21nIHQ3vrmAGAMY6iaaeKJ78gYSfm+O +zC5o8f4PLxtPfRJGymGUISrtIpBUTz+CVNAwQw0yrAeROizoUBhpDmUVrzlSfwgm +uH3H94jvi/IpoxUezsr1tS48G8ayaVlrPLflBhgX1NXzonA0KqGNdm5AwOx3jgZi +v5PYtz6xR1NtINnGm42A078gL9XHXmZowacGPNf7flwU9UzU1LDb1snGCxOgNq0e +ABZmd4TbEqMs1mAmn6ymzAIU2wBOStcC0is1MJ1j/NVVixETrx65UY1BTMlIojDT +bV4fbYy7jo8rtoLdnwFd+DdXna2LJ8qq+7CE35f9c+ZxLkdSfqADISwkhR+J/6TB +plRVyBc4eiPdSV27qPbjMOEHKy43+rHTsBILqIOQ0sBlMe8DgD8ZXo+PUM1Ptadn +PsuHwSflCrIz6AHgG/nORSDQf+7U79mejzi3fIjwYyYpSlBemTuW+z3x1xizN6PY +CK9zuZu8K9Z/F7GQmr3xBEtWqH2rd7TcN7fQP+zmG/v++vPlE38cQFWv5vVISxJe +U6AOJmXKNvDB4QR5CPsd6UWn/Nb1K3/MlDu5ymLq9JXm/GeokaGqHuaLLR20yN7w +/kNCmGr6r+cWAxpEZkmSBUg8/ps= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key index 17a1eadb3ed..924a050733e 100644 --- a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key +++ b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEA8ddEoeX8uxoMUafKzRviaUuUZDqolILm63wVmhoHuQN4eyLx -OIVWD9EQ2CEUEKU/1oaFbGd0ZJ5BMOSD4o/Eqi9cFfUn2Y/7P6iXjtVBV4JP74C8 -9Bia0s0vBmDR7kZUcUydWrrguVNzVeO/OOVTvnUDl0Sr+aIymi/Ht9KgjbPG1U5j -H4EXiGv5YDbSbFtCgZdUzwhMMFbJIzkgOGeAo0B+Zc1BkV98mCT6nB774jgrNGQt -TTY4RScfrf9gWSf4KhkGaD4gjxw0i1FVgJGJ9dvy2w7o4TBBAR7VFKR+bGKKJNgI -MwHkZRKjAJGi0pPAReBbBMsqCHPNSgI8lqmxv8LNjbGn8QK1C3jbQIlqizH98uZy -4qZuFbzjuFirgvwepm+OzeLsTrCO5QCCf+6bbyRTU+s32XqpMmEcoo8ZlQIL8Ki7 -SD2rtj1p3rvaZaATGjQjW9ebYfUo1jL3hqCZdoiCZ6W88FM9ESQQQvoyd+tkqJg4 -bnCNIy5uDQvglgpQy5Ohe99fF7qjasOh97+hNRcSsYNhXjPu+vnx1CBMzk+3GlkI -uaoQtoEXJ0HmEQFCXLwXGHdJWm7CRxoLE6VcyR28cQr11ph8T2NXR+5K19ySQIt0 -Nkwcj6JJ1CyoroT29zi1faueHOAp7HQHK7QZQSDpegUvYYp1Q69FlMn+0zECAwEA -AQKCAgBirops3aHQ3wIj16njI/uxJ4v3kg/z+a4sbMeLyA9aqVXiORHhFFdXwOF0 -r5q2aBeye9W5tEVEQWcsgjJmom+HWifjNdP67MuC4FJnkBFHmf7wJcH0gI28csk7 -ewpuhfgy+yZCPXsxZCHt7C69EqanUqwSb40VIeAwFksScTneWHtL0t6TnmNx4U0y -u2wFlv9+TbfneuVL3D3ax5akLWfKNajd9XFJ1oLKulwglGwHlQ/cIvxxQyyW2y4M -G+a+upaC5w88Pg2fF81BBAE1oy1+Sy+/v0e7i6sav/AGqlCD/XmNKzG9TopZy88D -jmm4L14HpVK+lAAkzsv35Q3AFKJYaB7+BdxSOxdJrq9RaxuCzZU00sFxA3F7PPkh -tokxrIuyl3s2cnJkHE294s4/1VnWx8UAvdEx+XRsypp0d7JZT5Rm3Q1spOYWS+Zp -ShlHtq2qdhjbqRLq4ur2WYCGWHqxGkXbhkPcO0ezstxQV5WQ/qLbSK0jflH90dgy -OsjDB14Pm7sxVOCgNaShhXZXvzS1dShug31gLDc7Zg8ubazKIpFdAGtLCL/LEUp4 -hb9PEWjshrfi1VQMSCguZCwuazO2FnkKvpP9M8WsxkZ7invxw1A7xX5tdeCfHaJT -yPar945pZlnyt3ROKzx+QLt7yjs4igNuLHFJ7LUL8jdfo3zngQKCAQEA+PoJ0Vhy -gmjA2XVkpuI8EVe6LsyCpPu58qqQPryzBApWZbLvh7RNVEpKrRl5oQbrgzMKxz6d -1dfCdLVA2t+aPLLLuvmduTJvrsD4pFuvmqdLbAGn7J+Q3IhdkANsX3dnfUeR8ham -0fF6Wcha6gsxvof1wTu6k10uT9mvwxEwkxWiJogjGMV7njHwiyXAO+dAUCUFG1GK -pEKcwqJYEbQLBC4dbgYJus7TiQ/pakAXD02z76GzGnZXxFZHbKo2JQpW+i2zhaM8 -Rk9CF4VruMj9ASWsc1D+tV+0f4JYKWk/fwCFX0rpluASXDqmVcF94xgxMPLcTkX2 -S1SfhPLnMNM6CQKCAQEA+Kmy94zk6xRSEhdqyTGKaXYL6pNTuYLClHomg+9ifsbg -LULza4rOWVv1gzdfht4q0LSuR9C44+GEdQlJ093ql+EiKGXOkO5DzOV3D0uJKaaw -jXmfhzVyRKAnG9oCFSoCIVpyVVLTSYsCk4YBG1+xC7jR5iCjSfDoXmXDNTbhvZne -CBPXe5Y16Fu8VKGKMOCuluRywkdGtWplGKIYM/BRpqypMeyUNf5cV/QnpnHpKpvI -3xpN0hWkVPM4mDxeEmxbaTMYqLKBvWxJIKwHlZsCoVrIE/sit4PFxD33A0gd9RKF -ysG9U7j9ezxv01GGeWvLGJJ2Rm9InJaraNdD0/s56QKCAQA9szEVqRaPjMgl27q6 -xJvgLF5eVJ4Xpq45Hd2rPnYajTlF8Kb9eUgkmFiTNxn2NtHMbKzCHZldsh9lbrmI -xBrj6eclCdD1YTNeGkJD8Ni+ubCuLmbf89chX8XX95E2w09pM4BOfyCOLIqGj2lA -UM/eCLF1VojxFglxDmBJNgxUlurdI6WW8PGmjXev9AYEklVUxyfR0k5KUJuqfDLZ -CEYmngr7BhIJMlInyEKu6v0L9NSfkPHin+s2I6+FlVTfrwVI8GRpvL8mKWQWkiDU -5TIVw7FNzoegiPwhETsp4AQ1Q/rJV7Y1hofmRT6NR+sbiRY/rB++YgYcOvdhKhTy -+5WJAoIBAQCKta8dpDy2XbmCWgm5facmOTFggxp7K1P7m0AM7OZWrf2zgdtjrCR7 -eG616ZeiFiBwF7n4WZgplk1bM/k49Fu+qVlyi4f1FNRBYedOYuZ6k399C9FPjSLW -4Tevs6bhw3M4w8lviypNpfuj62aGrJSIY2rDIoWz42zb64w/DQRTzZcrjlXvnNUp -brLZytrZdRwvE/RoJ3ZsEJOgcBtsLax+e2ZOcRgkHmXIsmvDz1sv54Lf4T5z6Jqo -wJxbX6FiSYucXWJuCFWTIuqQDSaV78hB1fAyJS8yi5EwbRhzfjEn2lNDzVsdbt4H -dqkyihGnfdfnNxw/IiixIOMxMpOllkixAoIBAQC3PapCqrPeQRBkUNqXdz55MLZN -R0O/pdkzIsRYbepAEGj6Ylvt+qgLYmsGfFdlhkPAOi9PxClRr44M7MzvJDF5bQlO -y/pyzUKZJ7aXG7QNnhxg/qZRgm0qj+RsR0rbH/Bc8vuahDgHAa1JbMXkc7O50+fe -fwebVZ8hscXVOeTaDNM/XuzmYZl6/lLTmpiqKt9J1F34J4O5mcGI4rS7kdMd83Tb -zcYPTbPXxgltIHIlM8ZFR7hSzdM6B3GcddPbWAFGOI4gY9e9a4sHtU4wH8O6IYlg -BcmSuIGANVLApRHMEabaEbBR/+QAzSZ6dhBaxG/QlqHh6iKSM29mSEUoSlZm ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC1AS/JxnAbDq37 +/7iOeeXjogYBWA/ld3rWohQ4ujLWcp4/aGs67LHr4zcD9OOY/Gti/WqY3qvVOURL +NjK2KB0xD6CHjohTzUH4gbSbfw4wYl2EG60n6ZIz5kURDypE1/ZRTUpRJht8vpDT +MgEr0dyuCDOganfhWLfxlMoWTyxcfWYUQIqubrpS6lWgHQKfsHYwF3knnbQk7Ptt +GlvjSgN9vj2/L8yaehzpofmV5aK5RMiin1GCNzg7vSKF07umTfF2eC36w7uO8LMI +EFOJ9kzp/kJu58BPRiI0AY2Ds7N3lGMsxP8+aMqwurdmdmRKtqQsDMH+4j10Ra4E +r4CyQgkncKwf208YITNL5hXqFVdr306d1Qv1m0lufbVfvojMjEEz8PoK4NDRCQq8 +DZxSdQy/Yk9H/dNs9ppJ8bgjPthxqbdhBvhZ1OJEdoB0CreATRiLrHHIafCfOQho +4XqDb7h2lJLVVsKlGh1C/G/TFXWr9dDm369Pm1jSGNsbLnzPLF/0D91bHyf5xZKC +JmDVuUiOYO/+Q0yMSBP8/AG9WuCU9iQszzvMAxcpXuEVThvtlnwwVJJIyI15X8Qc +C53kz86r+Y5qTyMHKXRdHPW1sCcG/3uIQf2wBcGbyxa9GsqSdGam3P+u5+Rw7Csf +Pv11Cmq9ajz+gM633nDpgcMDhwLBOwIDAQABAoICACWvipCbNUoc41KdaoKXlKqx +WEi9Wgnvt26qrkUlV2JY2OPgzLqm/2BanWDAgwaeP4mLs78CVYp+ct6p+SL99R36 +3/R5U58HFF42OzFI4d5nDItt675snhkaL3Ue0DEnOcyxp/B/W6e8UZD3T1EdT7IY +WQ9kO3N6skmlLfI5J8ZWAYPj8DvjJ/4NdihxAdPXvLPEn2AFMLWjdbHAlDMdfN6Y +VmjQ/euH0y89nyZ7ekbSsGdTfnf7IUUm2uI8NEyq28RLv17/k7RuHwhlxc7qrhbl +7lUDAYI/2u0zCZDLQGl8Rq3SkkYfMVxcc8Tsj3MS1dmqClYUKrFjtJibJdN3dkoa +/I8OUSG+OO8X1H6nd0Ht63RqBXYgR7y1+P8aGVuJYnNAdnj2iMmGulQmaXthlX/O +2lpoFJmhSZT1Ih00m29yIY8XhbKXddZvjg8zhVYmgk5KEZhN2+tF1EYTsBCtxvWh +jJ3O6TXSP+WQFByeAMl07hG/1NGa9hvgOktNCT3QjSosJWGRNj3oaZL3RwtUYOnJ +UOBzbPY00jF19/KGUMy/R8t1fOS3Z8e3qD9CLeMkr9TN0VBXFGYaMSrq+AvqTgco +uA16/ctdZcj80bwOYU6JqNqycW5S4V8t2KX4MhMf5bZ5dL13OVPhVY0ltKBEdOC3 +YBO5WvQ7l6pH5QEjWoPBAoIBAQDo1p8GDY62MMqxTUtrB/Bqz/Vsa7SgCGmSmRBB +PBfkxxARI28jTX4v528au1J5h3GZNs0gzWtbQW0yUN3hvf7KaERfBOAupg3nUO8D +731JL8jcX86UjS77PxzALlvNNONItFWZcH2VIMKJCeFRp79cVaHZtqHMqOcq4Lrc +BYjZyIcNVSMEMIO9E1ZJB0t6Gesa69OPenC0v9ywBF2h0YP9XGbCN370JfR6rJwZ +yvB29et91/agCorZBlctnRhRXYjQVqBMetpsC6tet3H/B9BfUSrQ81zeEJes/roz +aAYym89Vvav3Gi/P/NP96aEefcyG19cnTTXskeSkIkpIEG+bAoIBAQDHApYSxXKK +iPsakBno9OUA747YvyuBI2Uh7+HKGbM7sVyl7UhwduI2qpxG6xjz5+NOxJe12XQI +91+KAciuKNZ3WguxtgKy5CFLlC3syV5nuvfTWwt19qg2I+EiXT1YQ09LmhG7M1WW +xwj/QI770YlbVW8NA9pB40ojPRUB6a9QynGBvj0fTdr1IDs23Eb11s/1f5pPAfmf +ENehdZG8aTuPDKbdL++yHrg2n+v/aAWS453eUvE/evk1kv2n0hqNwJomn/e7MH0h +DFfdTHNOaT9jS8W/6TvnXTnwqCPofbVdJkwIpsyqFP99JpLv2DkjWBpnER6K2sN3 +4tlFutH3fZ7hAoIBAHhQluHg1ZLNiZ+uyj7nXlNDXJWG3lInpq4LC/w5mjsspd/F +SKOcYo1McPwGOer0kz3RtJOzsjr4hv2ODemdbPCYimPzOAjC48JY7vlAoqPT3l4J +38yhW9qWVTv1eJiEMMIb9XORTkNmJz/67ARuwxJqaXT/y4He8X0VnVkePwYSeb7o +9GwwyZJFULvx2g2hIPzLSYoFggHUdqw5xUxd4G9mdO79EEwG5NsTIKKvXCyMjQB9 +PUKbz/2vKhM27x70T0Mg0DpU7VMRgJ8z+eMH3aOzZIWOi7lwJ4BMLFlt8m3qV2v9 +08OfM3+tLLozJH+GD87J77KuUiQJi3VJBstQOusCggEAbPNOg9JS5eMeTamMdcv0 +v3FnK+RlBdsB1/WDEGjHhiuQhEqGeQOszq7SxcSZsZZHDG12fRkifHIpfN2DJ022 +NDs44b66dYONctoZ1mNb6sAiQzS57ofuCU3VSEHxfqvC8qnTgUAWYGhw1YVbgiNx +5m28UUzFpYKwqOAc6yIrV7itMZGb1tKBx+cwY9Vuh9JloYqNSSnUpi/LTVqXQaX+ +9E7B36AJwuMsiMM+qWIigwtX90pDRzSk0J67fa/515cVDYZQUsu2BWzDgaUED25x +RIRMK8edgwm32bfDiNmhe/CmeaoPrCG4yaRfly0yAtaOBdUKeoxPrQelWZZwsuS0 +IQKCAQEA1bzyZGNAf39eJ8munj703lTgANCjSqKMoHb8T9Y42ikOvrM0tH/0gOTs +43CHNI8gxeJXrjv1HpUr7kfkIX5aBvRGoG4kxS4uodQUWB80KW66esbbU2XzpuGO +uB9jjmY42EhT7wXNde9PWdu288YhVHqqIXHQMbdjfp2HjQGUnKuwWVxNSDtFSeC2 +yTM4cqkUH5tkpH/2KpGgZzEUN2MtHSt0+kojkVei/RJqRqm0/YnN9h0j8wAr2DRx +6tEQIBg6JR3dqYLmOTrTJl/6Quov65hbAR5gKDCaxw+jBp68hxs+SxOCBLIKgzSz +UhU2dLCAXnNdC7uFxDlGq9G5/e6SHw== +-----END PRIVATE KEY----- diff --git a/qa/integration/fixtures/webserver_certs/generated/root.crt b/qa/integration/fixtures/webserver_certs/generated/root.crt index d5653f0e853..23c26078be2 100644 --- a/qa/integration/fixtures/webserver_certs/generated/root.crt +++ b/qa/integration/fixtures/webserver_certs/generated/root.crt @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFejCCA2KgAwIBAgIJAMwwXnXAAUBzMA0GCSqGSIb3DQEBCwUAMEoxCzAJBgNV -BAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVN -eUxhYjENMAsGA1UEAwwEcm9vdDAeFw0yMTEwMTkxODI3MjdaFw0yNjEwMTkxODI3 -MjdaMEoxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9u -MQ4wDAYDVQQKDAVNeUxhYjENMAsGA1UEAwwEcm9vdDCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBAMfReGxhC2tJH3y/4knTrBSrtezJb5fDs9Xy41Gdr5ng -YEQXi7JQb/MXvGst4xxX77kbBv8uo1egL6gnvZfUqbsrGS8IzhBWtRET+yaYw2ez -wZW+d9RGjsFTOGvkSpv55/VdGvtaXeMeqTbuyYZSzakNYD7AH2VDCS2q+IdF/ezm -ocOki5faj6A92yxvbHUZM1jS+Ah/v2yzJOAkrhT1XIBKyaG/g+woQxZpMS0PjpKY -K7K9XPIkU/Z2gsQkRfDQAxXTqba4Bl84/aA+aLTYqib/5WhxVuc8vjk+7g01y2qX -5aPYs3L2LToUfwPh/6leHx37d7YouPp3X3CnPyl15r20FjTFKTV6Y0he3tgmWnjC -BP0/qlCYeh1mnKFQSo688fw7K25dPhcnPctdVp5Xr6jz8GBaGgCu5hEx/QI7WzXy -qKdyyIj3gEQuHyu8RUrTUA3uruPa8TlouAu616yNNqw3EzrEjmWX+D4HRq32ofkK -FXqFxFurfzxyVJjX1P2+VP0h8fPoZJzBwfwm1hFxLpROQXVBX+b7j2S4kv/Lp156 -mV0frWXlYwTvtM+7PKONnwSIOkiTic8Zv5TcMLoW6p4RNY6b0SDzt+e90h20OeAS -R1gv7VhGfR2JYMd1C0Ke8D9i2FebX9r6T/BfRMRJBNKj6wr6OFvhz+eImNuiAaxn -AgMBAAGjYzBhMB0GA1UdDgQWBBSWO1UVkFl5LOuplY08xBLcfGyc+jAfBgNVHSME -GDAWgBSWO1UVkFl5LOuplY08xBLcfGyc+jAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud -DwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAHquJcXAZlMpY3z748dmIfKGb -7F4fGgTwGJ/9PLIZdZXDRIiA/GZs9IGA+Kykv/95jJsq1ZVN7O7UoBqbTxbsddoU -EhBe3BKaJtkIuB5anRmw600lZlIFgSU1uSXyZoDBD9LnQnb13f2UQJKvzNbvoisC -f23JIVCQhGUiqVVYKF80tWlbTpnA+pIPvaWpzSI8FEjz2NGmol948aEp4OET5v/R -obWImc/X30t5Z4wO620DCNVhwx+zzomAio3dmQRnnUrVT9IwKxFmEA+0V4O4JPUz -YQMyuLuoqrvywz7JeV3GvCy+yI5uoF7u5m7e5Kh3A1G3xfbw36M3fYAQNhh/5y01 -ySimOmx6CXoX60z31RFlrXgAycLWKixkjllQTMCMvtcFOszLSpP7WoB2/l7s+9oz -iMEnjwvtUcXJ0rRHr2ENmyG8BBPKZZhKp2b/kumQnYa8Vj8FVFXGijLNF2QHyIWr -jzc9ntZyUb2LOWA2Dwtk6BxgJrKfx6N4n6itwd//mzCz0GQZEVPwlLUw0qO1qt5+ -dSP5ifsecUSsXge3nl+pZmS60Jo1/sF3QhFiFbKMRi19SqvI2X+vDwAbsS05PMdt -HpN/JLfaeabqjoYeEpCGRm7ZSIR0Os9c5qJpXo6V9kOUbQKMNwJ075pPaQEE+2Vn -IQqdInXs3f5UoI5afJQ= +MIIFhTCCA22gAwIBAgIUM0vS1bRQQCJzcfsSQBT78uhQ7lUwDQYJKoZIhvcNAQEL +BQAwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x +DjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MB4XDTI0MDcxNjE1MzUwM1oX +DTI5MDcxNjE1MzUwM1owSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD +VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MIICIjAN +BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqtMGs9xm1Z6/B0FrYwxALZmzXk9E +HwS9mfmPbPUW+ZceZ8LcF9deWU+gtBwEZpV13ILhwHdlMacsnW0iriQtZKmX5/wU +Eg14tRj/vvDz3uOXv7dQ+h2VghlFvQD7iTxRt39a1NnbrKMmWQ+vzAwGOcuWthY/ +Ec9JZdYbvsBCS+RMCcQrd1E9xHjV+HYIdS18Hr5TVxnfW/Flm931i6tDsTFXvMwk +dqgCuajMvEKR3yhDx6rFxxRN9vIezbyMuVBjNhYsRVvv8FNQOKOMIadjb/vRrkMK +VgUL4WYi/liqjSCIV1rC+7v/DnvXPOCSaJgNBvB0zMJNPxSgTe8iH/LVnEHuCEu2 +MSnUVL+AApKOjiYK1Wr4h9ymllnRgq5XlsUwGnFkcaEkGO1UaHNFs4z89dr32hye +RO3T7B5ChQVvjbYK7piDDto69OW397i+yHVLxD681+KrZuKOeMZr9iFFZaFLeh0V +7l/hahhIlnZGdK5hGWIHZu+c+kfP1Q79drspE7Rr4AoMxVInxce7qV3lNMUp2v1a +0EjyFJ+iJquIBQdJTtX1rYWdFA5BBA00DGGRbIYiOheKOEnTcCsUevx4T8+2G0PM +pyCDTy7rj4pnXLr5ziSsfSww8JnxsU1IOijJ4L4b/qGCJ9JfQpslfgLBfwOJ94ZJ +mv7qjgAcR1HrV6cCAwEAAaNjMGEwHQYDVR0OBBYEFAHZXM7zqCUB/XBvn5uNbxg/ +FmWlMB8GA1UdIwQYMBaAFAHZXM7zqCUB/XBvn5uNbxg/FmWlMA8GA1UdEwEB/wQF +MAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQBslT8qF4mm +B6nKUPREE/YP8Q6nTXgZ5XhhHsBAHNTP+U2/lNjfQdFUva9mHwJRts5w59phfMpw +5gTiXJqdTzZEIEN7cdKX2aNL7id6on9j7OoJsQ81MzhUu8rOGqeCYFJFlCMXeby6 +VgInIxhIoe2RrM2bJYw8turWd/+VAa34ehvS3knvGGUTEr/8sqJ14hwifpRl+YB9 +iC1CTL0k00YXJ1IFC6sUaVya0KCRzd/J3I9XnUKOvHxSavA5QosUoWjQde+lJM+J +8sJHudaM1I2bn+8M9jqSO06/SNMdoNhbapKyvRV2DUwpMbe4WT8Ufnvbxwvw7vnA ++0E7X10KcaLn4+0tKhmNLHb1hjbaGryHO9ESSFGa7uMeiQYo0OQeuLWgRb6mOsZr +x/VEuytnK+YwR0+vXBCC7PSxuZ7G/MPv374w/+6b+rsrqz6o5JdgjTu27OISbNv9 +5IxA9+DfNvqyLvSQ72M3W6BHAE5BT9/JsTppcJ0KXc1Hkb9dJEIScO05FdsDC+SH +XvTmuXCf6G5XkrLldshntu1M2MI2Usw5ZK2RP3H25F/AOPxQDim5awMp7Feo0JYC +EXJedqqTtc1M7ub8XVvIHTaT2jNoa5uuV8ujyiaLXZqZty+UUkXUFHx4UDDWnOrz +mHyg8QosOZgiGREyjq0FJNDr9337zIfNkA== -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/root.key b/qa/integration/fixtures/webserver_certs/generated/root.key index 06006400851..29cef4f921d 100644 --- a/qa/integration/fixtures/webserver_certs/generated/root.key +++ b/qa/integration/fixtures/webserver_certs/generated/root.key @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAx9F4bGELa0kffL/iSdOsFKu17Mlvl8Oz1fLjUZ2vmeBgRBeL -slBv8xe8ay3jHFfvuRsG/y6jV6AvqCe9l9SpuysZLwjOEFa1ERP7JpjDZ7PBlb53 -1EaOwVM4a+RKm/nn9V0a+1pd4x6pNu7JhlLNqQ1gPsAfZUMJLar4h0X97Oahw6SL -l9qPoD3bLG9sdRkzWNL4CH+/bLMk4CSuFPVcgErJob+D7ChDFmkxLQ+Okpgrsr1c -8iRT9naCxCRF8NADFdOptrgGXzj9oD5otNiqJv/laHFW5zy+OT7uDTXLapflo9iz -cvYtOhR/A+H/qV4fHft3tii4+ndfcKc/KXXmvbQWNMUpNXpjSF7e2CZaeMIE/T+q -UJh6HWacoVBKjrzx/Dsrbl0+Fyc9y11WnlevqPPwYFoaAK7mETH9AjtbNfKop3LI -iPeARC4fK7xFStNQDe6u49rxOWi4C7rXrI02rDcTOsSOZZf4PgdGrfah+QoVeoXE -W6t/PHJUmNfU/b5U/SHx8+hknMHB/CbWEXEulE5BdUFf5vuPZLiS/8unXnqZXR+t -ZeVjBO+0z7s8o42fBIg6SJOJzxm/lNwwuhbqnhE1jpvRIPO3573SHbQ54BJHWC/t -WEZ9HYlgx3ULQp7wP2LYV5tf2vpP8F9ExEkE0qPrCvo4W+HP54iY26IBrGcCAwEA -AQKCAgA4oYdtv/tKhsjjxEZQAivNF/VRjK2R56c6RcJjSTuRcuQVqE6xVNbFoL6H -6dNqHwoT/jL3c7JbjUy0bb7wHxeTsl1K1UYBn3Fly7HseHM5GtsMo5S2luCyrfFZ -srPqjONOyPpyrmPZkMEIjpiS9zjOZroWTsEXI+6gAMq7G/0hfSPvnNzrRshZ8+4A -I/wrG99Pl0swrXQFTKhw1jR4QCndOlT3AJ5U+fw5h8qmKLiRl4sa90w2XFFnkqLx -EA66d1nIf9qjrHgx2KcZGVb3mkhvEStldGna035eHxW8rLhgF3i//RSuylJ7GX/+ -C7k3uTM8fh6pg9UZN2fXN69RAtdH/8wHeGRpMVfTyFA+//ChEr8pFDudheO9elZx -ahMB9Qm+QspfJuGiA08jbkp/Kqe+vs53SXvpcWB0DCba3Y+PzKlJuQiuGJbjC0x/ -9LJjBZCB/8VyWCz2Pcg5arUyfiHz9iJIC6vfZRMwrrIvD+GQe7jTPT3bdWNeLpRQ -kVjy5AjYipblT2pSgsh0iOBwBoxqh8z+NOUgsIjcLWlMXLStQpNILuCc0qVUDfsI -LCt/BujptkezwFeicmjf4a+uavoUODDysfMxbXwyWVCPoHmcVUIj00Nib4pv4hDw -j124iEyZVy84rCUlYHb/uGpwSqE0Hw0diBdx/g6vomCMHCxLAQKCAQEA+Q2ViAWw -ekNH4wYLNDFZEhcdP9tPPP1z9OE2N6+BgpTqfiJMBa7tVMEd0HdE38GZO2Q39hpD -Lai0K8/kqjckP6CAWRoT1vgSiLVvdn2NjSKhidostZ+GZyBDbr97EBq3Q5XGGdYZ -021gQ+p27sWzwNCQfewtKeLbWWRWtgQPmrP2kwaF+xEYcfE08wy41yvypyWFmAm/ -e1KdTPiwSXjDHawUl2/Cp6S5YjvFWAMwzF7BIEkGcRqjvi4eR8+i1NEJgY/7/354 -B7/wxmPWTaqk67+8fRNSzkMR6KNafeEd1iS+TkcdF540D2kM9GJiBVBYvMboX1jF -dcNUXi6opbQqJwKCAQEAzWRQj7LkmcnDAs6HM+/UMbQ15sUHrTqDfFF85XA9/ZYA -ZHWTyLRWN5TCiRHGK6vilrVBqzYkQD3sMMepJwklhAue5SnvXLiostf61F8TzXmX -uQ0Hq2goyHGBTa9RZCD48mliuPHdjFQY3WzXGie9XvkDat8zgYtZi5ODMO27mSTc -8TnFjZ/Klp4mXgfoOzv67yTlE/DwmGQT0/HxtA+mwgB1WfKNuQqWJ3c/KKcIJ2G4 -Gf7Y/9ketF2i3pdkri1UBtnhL+/t2n9QLMjtt08hXWfrrJPfHQdAE/O4i67mbl1i -pq8OazK7YGFp//vDCCuvrfQUo+bA4TFZ8+pfIqsTwQKCAQAXfx+XaKoAru5OTdiq -wOkdWs7oomE0aNkSxlieIlG+pxwEcGnps1oyv9F3RHlb/uDdCJR7BHlICn2NEJ0L -Lg6ebh8NyPcMNfVRroRokmnTYewJnGrRa857VkDmZuxOVabqmmvJLjjbiJsnbSxx -VHej/klTh/+uzvdQ1LWVkIaEWuwtbNUhHbjHrzEGqTGbI0tOV6zl7i2BjgbHJ0NC -e0E1QHHk46dit5jB2s/AgPSOo3Xm/Ny9uPyvyZwTvFQpbdtOvO3SafUbDhHCNdMX -Gr1afIlk53jNeuhqoVa9P4rYFDlcGzt0afmRZqDRQaLry2vDy6RKso5WkOoZxJ8m -mvvLAoIBACu03xfVH6RCNQdR35+Dt4nkKVw0LVAFlZlM1eEVI3vlAaxfANXC2l31 -f/gfGiaM6yja5WCDou8wviQbOLmfzqXsuyFU7SSz/uIf57ZtjQ86grjUNv+YqKUO -0k8vsYg3JAKhvmpQva5w7uGfmZ5tOnUNVFbXrGQyvRlgiJhO3o7WM1X5zzo1+KxY -dx2nXRnNx7ZACadWrDwhf76oS6Dfe8LoyhAb92CbChzX2U7TTH7k4oJVouO+1jPX -gSMQ4FHYP3s9vhevXNQUJ4GAOGW7pwPza0lMJjxNIKueP3UqAMsXgDisI8tX/lui -HyZj8rq4YSgJOqMuJXlGQTgQdbIw6YECggEBANr5VriGXg/bynABEpz2fnYJsDPz -syxNr6262pYFhXHsuawi9RtSjndNwrUUnfEnFddSBHXzNi2BkWLXtjGdXKzF4Xuy -TTq82gXNMAxGETqC6hmof2IxyQjomoXxUvlJ5IKEoDLGRaaO9UhlG7GhsfJth5GN -hgCsij+llcgKTuJ3HJx3ThLO08wNwk4lTt4YxDdN3HcyhK4YZ1noH5nx68tjnC8u -Nd1UAPFhlPhUH1hnp9zZaFxaSU6MvnDVUIBayAMxpjQXsn9GmdG3MB13o8yAeHkP -U6MHvFri1MvZ45rd+DHresJit22GePxH/9WrYfWl37yWVSwTffh2RE0x4F8= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCq0waz3GbVnr8H +QWtjDEAtmbNeT0QfBL2Z+Y9s9Rb5lx5nwtwX115ZT6C0HARmlXXcguHAd2Uxpyyd +bSKuJC1kqZfn/BQSDXi1GP++8PPe45e/t1D6HZWCGUW9APuJPFG3f1rU2dusoyZZ +D6/MDAY5y5a2Fj8Rz0ll1hu+wEJL5EwJxCt3UT3EeNX4dgh1LXwevlNXGd9b8WWb +3fWLq0OxMVe8zCR2qAK5qMy8QpHfKEPHqsXHFE328h7NvIy5UGM2FixFW+/wU1A4 +o4whp2Nv+9GuQwpWBQvhZiL+WKqNIIhXWsL7u/8Oe9c84JJomA0G8HTMwk0/FKBN +7yIf8tWcQe4IS7YxKdRUv4ACko6OJgrVaviH3KaWWdGCrleWxTAacWRxoSQY7VRo +c0WzjPz12vfaHJ5E7dPsHkKFBW+NtgrumIMO2jr05bf3uL7IdUvEPrzX4qtm4o54 +xmv2IUVloUt6HRXuX+FqGEiWdkZ0rmEZYgdm75z6R8/VDv12uykTtGvgCgzFUifF +x7upXeU0xSna/VrQSPIUn6Imq4gFB0lO1fWthZ0UDkEEDTQMYZFshiI6F4o4SdNw +KxR6/HhPz7YbQ8ynIINPLuuPimdcuvnOJKx9LDDwmfGxTUg6KMngvhv+oYIn0l9C +myV+AsF/A4n3hkma/uqOABxHUetXpwIDAQABAoICADul3B1JxI6rE8EQ0tKPIInL +tZoky4CZQytUvQQqwI7+eZt5qyiYB1XgPBkn/1Fitvj8qp9uEWO5IBcIWKe98ivI +HEbg+pEjKxbxV7vwnvchOYe0Fw5+7gmzkWphwXM2IMKWMb1JJS+u+GD43UW+nR3p +PQLv2a36V4K7tIxyzbN6UCLI4PSU5aCJIQ3VCTcBXunNQ4SSgC+2pug6JNdUQhrJ +Q/2kusbD+8+gZX0HMbBeeuxpcQ4GxQKJoL/uKqU6sKneLC873xhnvwdH9FJ8BHjB +0YGr3xwSPwfBhk6R31vQFEuMYbFM/86q54lWUUk8brARnncxEZAJP8B208+cF4IZ +KqHPP45r1GwWQ1G8i8IPFmLhmiSZU47NiPiYglOlBsSGuG2zPLHJUWZnetEmJgFP +5h1sBL8v6OA79Y6dMWXgs/Wdu6jOrcv9ZBEPZskqnYqZ/AQknqgIKggkvegksjFG +OoGu6slQ9QRDm09bjIEvqaVKNS5aDhmGPmOvaCpmE571KsIdWFMOIBIvJjmJ34hL +6qoLJ2XoTM5QRY7ZyA0a17uEiXZCDnd4vggbEt51CCwKHkY0+WnjzT0MgU9C4Bzo +B0a2idW8UT/yGxPz1uGFY4z/RJ8kW83MwpGX5dNPYFEq/1egZVehK56saowetPlJ +LFkoJNSA7ZetxIUZvD1BAoIBAQDuPIc9WPhT1Z17R0g/uYnQ94H8ZbpPz5sqzzGw +gXTcYDIuEVixUlDE98KDh7BswGoh9O/PSGKYAWt4FhEzmUUPGSOV8U4vnPNB/tud +quVICCaVA0qYF5RfDBZkMxawaP+0wjhiReqKMZZstEEe3RBiy2uEnkHfDXmXlTER +plF5wFTc+luS7fGx6Sl5B4rSWrhdmHAZC2Qep9SOBjNPg/BwvlKthXlDUElE/eP/ +hOOO4upQBBENKM9iRZfmNW3JU3NMRZB/SFVi4nfAcKD/k8wlp3Jf3US4E7WJxhzN +WIWfYpfEHFgxxCLBrpS2axlRe9rm8h7uWLnDspVgXl5WdnEZAoIBAQC3j7s069A5 +Kw+/MbPWMZ1tGZGKnl4uBKq3LZAz/i3/mzjhleaOEHYoam76ELLLIulblc5mjHwf +8he1An7L+ZmTo0ezwT72WHBmBAjH5Nn86BgB+3DqEhed7pSDtwz28VzV5XtwH7Ug +enloX6HqxDiHyHTyo1rcERiOq2HsHURUcu+yawl/jOadv50qln0j2WVpGeAQo4kq +NwY+OUrZGOaCtd8RshX4afNKnDU05Pk2e9eytqfoa+S0/86VhkZ5cv4pc1SaW9Hf +juUWNFeTnCb58lk7PN6W+/WQaYdn6don5yrHm3ne+ThoOXlQmbjCQ4m7P81EVJ1U +bsetbqzRBWa/AoIBAQCXkE6Byv6BZ19/NIRUdlelaDF8P8PeTrfq7lIwdsvAADss +b8C7XyEfjnhjcYneRjO/UrLWbri69/xFM9TRvOrghh/Lnu8c/38LN3bEwhqITxcs +qiFkzWuJZIlCSfZfWQ2FFCMG8nAaZx3KHotndUCdecj3YC+MFmiK9VF4ZxVineff +hi+INHUVuL63zwKUfT7Z3MdER6j0j9V8XZl7Nvda4tp6pnXVS1ix2cyZnROgreBf +85Zdht8ZKClRPiCQetTZa1BYic440NdayMzEjGTOd88eCglUCsXt1uowAedQ1Uq0 +a0v4rU1nZwgZXd2Hp4JGdzcSb8QPKqg53Q+rdkLRAoIBAEwgxBdSpWzQOxukVV1D +/nI85GuE+rJDJToLH38wr4sOqSyFqrTIhviEC7qSKlStCfL90jgdk2z8TnC3tDaM +MfFVCmu2fZJVY2grzCh69R06dJp92Xp6Pmk1hUqLJElWGJZtMiByRgBOSwPKhbCD +6J4HuNjYfto+DN0+nsH58VjEwjxQwAOyQAK0cU40U3oGs1Q/cvnzmPN1FpW/1Rbs +iM00+GqyXQJFjq0m07o4LFM4UWDA+CogdR3L6bWBlsDPvDEcr73EzunUrZXj515E +qLWImRFEUVx06ocV/R5PkjZyXP+jr4uV+JrhGzseRc3rjLen4k2wef+GYjG/cNTP +VKkCggEBALOgG98jJQlbW5GD/jBkizeDiNy99H+Cl7MzmzY2PAtpfugeMbBYceL1 +2weWpWbVpbCp/Zkksqh0kYgI45phx+eZo6tLZd2IOM3A3AbnkgVDK+1gNSb6s7fq +g0Llaz6wAHrLjTf5N5vbGJNbzez2UWCJR8boxM4+Q3pcOSFQbPv8rx09zMNa1xaL +KuvHT6xTab5q+qz4BiVtrx7ZWqakOlxO3J5chYZUsxqoH3jvE5GzY4yi52ZPyt7v +rHZbCHclS/MS5pBUUCTj0Q0qCfiT03m6nP6YgN9w1rQROe5EzqPCMfQlVKo8ayQB +tiYXTiDTadKNODupVYUpbHSfZcFvRhE= +-----END PRIVATE KEY----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt index f4c8cf7f5a5..c2360f241e6 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt @@ -1,68 +1,68 @@ -----BEGIN CERTIFICATE----- -MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzI4WhcNMjQwNzE1MTgyNzI4WjBVMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTAzWhcNMjcwNDEyMTUzNTAzWjBVMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxGDAWBgNVBAMMD2ludGVybWVkaWF0ZS1jYTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAPHXRKHl/LsaDFGnys0b4mlLlGQ6qJSC5ut8FZoa -B7kDeHsi8TiFVg/RENghFBClP9aGhWxndGSeQTDkg+KPxKovXBX1J9mP+z+ol47V -QVeCT++AvPQYmtLNLwZg0e5GVHFMnVq64LlTc1XjvzjlU751A5dEq/miMpovx7fS -oI2zxtVOYx+BF4hr+WA20mxbQoGXVM8ITDBWySM5IDhngKNAfmXNQZFffJgk+pwe -++I4KzRkLU02OEUnH63/YFkn+CoZBmg+II8cNItRVYCRifXb8tsO6OEwQQEe1RSk -fmxiiiTYCDMB5GUSowCRotKTwEXgWwTLKghzzUoCPJapsb/CzY2xp/ECtQt420CJ -aosx/fLmcuKmbhW847hYq4L8HqZvjs3i7E6wjuUAgn/um28kU1PrN9l6qTJhHKKP -GZUCC/Cou0g9q7Y9ad672mWgExo0I1vXm2H1KNYy94agmXaIgmelvPBTPREkEEL6 -MnfrZKiYOG5wjSMubg0L4JYKUMuToXvfXxe6o2rDofe/oTUXErGDYV4z7vr58dQg -TM5PtxpZCLmqELaBFydB5hEBQly8Fxh3SVpuwkcaCxOlXMkdvHEK9daYfE9jV0fu -StfckkCLdDZMHI+iSdQsqK6E9vc4tX2rnhzgKex0Byu0GUEg6XoFL2GKdUOvRZTJ -/tMxAgMBAAGjZjBkMB0GA1UdDgQWBBR6wQ0Is6YucsLFNwdZ3ANX1jrgsjAfBgNV -HSMEGDAWgBSWO1UVkFl5LOuplY08xBLcfGyc+jASBgNVHRMBAf8ECDAGAQH/AgEA -MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQUFAAOCAgEANoo45TDx1lfglAmJ -8nt/58y7fm8trxmrSQmDAIuXZKqATNsEw/+VbIXqeC9sT2lWY9URFmmFm505JP+1 -43nM68cjfAK7ihLcGS7XwQMtQCgiQzJKeZFOx/l30lN5rRje5HHbSIF+uuwO987J -LNYJ+rS7N25lB5EQY8satuZWMVtSZ7YQCMkSJve7KjfrbE3kORjHs6jn3bdyuWAr -oApgKJgCUX3JSPZyH3T/xakMneZNkYr0CC1kNi9wL4js5I/nN2Z4er0N1Zt6PIM+ -hIJcW/Qv0BtI4bzkNkGSehm+Ofr+njuqWaaQMF7kLPTGDg/0SN63Y+X59CJQMnjF -+oW0oGm30S3okFpgXSU+DDxNPlh1j3jTMoArPvIaTceImfmsBXU62JRXQWaAr1jF -EujYbTJ147/7NmsmnskFO37Mp35JxL3ELWfMeBAL/HC11mzUbGFhtvtFqhQMWMbt -5AAR9ZUaPO8xTX6Ag8/8MXF5wFRkJBqTNn6GaX5PUrWhlAKpGuAuuR1NBY5TGUHX -NcFu+bf5JtjhgSUGB0FuZfdZvw46k30NLlT39NbXayY102XYfrAsl0X0TNqjQua1 -7j7FqYrAALQVLfYeKrE0nKaH35nU6Kk1Iy42CbB71MHqcg7DTjn0MvTKZcnQ8pxv -RQwP+BXkzwfWWEp05rlVeKS0IjQ= +AQEBBQADggIPADCCAgoCggIBALUBL8nGcBsOrfv/uI555eOiBgFYD+V3etaiFDi6 +MtZynj9oazrssevjNwP045j8a2L9apjeq9U5REs2MrYoHTEPoIeOiFPNQfiBtJt/ +DjBiXYQbrSfpkjPmRREPKkTX9lFNSlEmG3y+kNMyASvR3K4IM6Bqd+FYt/GUyhZP +LFx9ZhRAiq5uulLqVaAdAp+wdjAXeSedtCTs+20aW+NKA32+Pb8vzJp6HOmh+ZXl +orlEyKKfUYI3ODu9IoXTu6ZN8XZ4LfrDu47wswgQU4n2TOn+Qm7nwE9GIjQBjYOz +s3eUYyzE/z5oyrC6t2Z2ZEq2pCwMwf7iPXRFrgSvgLJCCSdwrB/bTxghM0vmFeoV +V2vfTp3VC/WbSW59tV++iMyMQTPw+grg0NEJCrwNnFJ1DL9iT0f902z2mknxuCM+ +2HGpt2EG+FnU4kR2gHQKt4BNGIuscchp8J85CGjheoNvuHaUktVWwqUaHUL8b9MV +dav10Obfr0+bWNIY2xsufM8sX/QP3VsfJ/nFkoImYNW5SI5g7/5DTIxIE/z8Ab1a +4JT2JCzPO8wDFyle4RVOG+2WfDBUkkjIjXlfxBwLneTPzqv5jmpPIwcpdF0c9bWw +Jwb/e4hB/bAFwZvLFr0aypJ0Zqbc/67n5HDsKx8+/XUKar1qPP6AzrfecOmBwwOH +AsE7AgMBAAGjZjBkMB0GA1UdDgQWBBT5X+u4LkGBED9seTSNYgA40tXJUTAfBgNV +HSMEGDAWgBQB2VzO86glAf1wb5+bjW8YPxZlpTASBgNVHRMBAf8ECDAGAQH/AgEA +MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAGb6iGwxKXTSgxP96 +4UMtpKYFGykBuajxr105jFSjXArhjj5/U21nIHQ3vrmAGAMY6iaaeKJ78gYSfm+O +zC5o8f4PLxtPfRJGymGUISrtIpBUTz+CVNAwQw0yrAeROizoUBhpDmUVrzlSfwgm +uH3H94jvi/IpoxUezsr1tS48G8ayaVlrPLflBhgX1NXzonA0KqGNdm5AwOx3jgZi +v5PYtz6xR1NtINnGm42A078gL9XHXmZowacGPNf7flwU9UzU1LDb1snGCxOgNq0e +ABZmd4TbEqMs1mAmn6ymzAIU2wBOStcC0is1MJ1j/NVVixETrx65UY1BTMlIojDT +bV4fbYy7jo8rtoLdnwFd+DdXna2LJ8qq+7CE35f9c+ZxLkdSfqADISwkhR+J/6TB +plRVyBc4eiPdSV27qPbjMOEHKy43+rHTsBILqIOQ0sBlMe8DgD8ZXo+PUM1Ptadn +PsuHwSflCrIz6AHgG/nORSDQf+7U79mejzi3fIjwYyYpSlBemTuW+z3x1xizN6PY +CK9zuZu8K9Z/F7GQmr3xBEtWqH2rd7TcN7fQP+zmG/v++vPlE38cQFWv5vVISxJe +U6AOJmXKNvDB4QR5CPsd6UWn/Nb1K3/MlDu5ymLq9JXm/GeokaGqHuaLLR20yN7w +/kNCmGr6r+cWAxpEZkmSBUg8/ps= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJQVDEL +MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBVMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxGDAW -BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yMTEwMTkxODI3MzFaFw0yNDA3MTUx -ODI3MzFaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz +BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yNDA3MTYxNTM1MDRaFw0yNzA0MTIx +NTM1MDRaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz Ym9uMQ4wDAYDVQQKDAVNeUxhYjEPMA0GA1UEAwwGc2VydmVyMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEAvkyxB/z2frDqa1F3lDqkW5L0F5L0sGZiOnzR -0kgPsPtxIjt3ntm+8SZAL7dJ4VSmH7DNo/IUtwent/ypN9b7rR11kl3EDcCoISPf -2VXqvi1cQou/M/GSZPDmZMsXW6kVWzZydOT4ub7sEbVtp+J4GhRkR/AZDLioBCme -Dw/GMg68+BHguPZL8c5oi+6YkhgLBvua2kum5vYMWkgATGneSZj6nWxbNRryFgq0 -X5qRrB7n7HQYjT/ki412EjbnfSpqfhcOAtu0RUI9XNx9QPx/s6PIxtcd3lgRfeB+ -tdPnDp4pQZGm6ySUSjZV+38+tKKCX+PF/u0gJWpQwA2x16lvIafxDNFd1DIr2bay -RC3vmmPMu58+B1hj0PILwnZcozBFMo/oii6m9lURUYuCBcHAKOyeMnroIXHEQMAD -aj9fFBGNAoB3gLT1gvHrrbkRmQ4b7pE3jDYR7kn4VUUxjIrfX5ucthpSIAqYFiRr -OmKUtTpwhPcRogOP36VoVM2bK4ShevhQdRZ5vjo2DggweIfIg0UBdLSethvct/hJ -fIrmq4JgUMwfHfW6HYR3Vn5T6Eag6+mwhlMXbg18D8r3XmzuAtMEruPJKdV3b4Ad -oZmnuYGvGBqHmY2fBMwdtnd7SJ3H3m+MPxF6/1Fks0D2JgVNGi9///nR6x4ITvjQ -joDODBkCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA +9w0BAQEFAAOCAg8AMIICCgKCAgEAsPeBqLtAWvBZmAtNciN1enkIgE7ffPe/8Qkz +ZK1so6LdrkQsE7v0HG8j86AdEaBTSiBfwG1+Rr5ZGpHwrPlf7Zl4GHb+v+gE3gkT +wponMN3wC8XAVExu0L7JnQBK50YEEI6/EfayQ9b80E6j2/FiyVj3rxt8MTwCAGdh +Fjb8Z8WSxLDelk9/2njOj3Swojar2QR+Yxs0mr7obOxB+ZonDSBJ21HK5xI5rOwa +wMSzyoHmW8Og7Z7TrGLei3ejvVy1RZ+pF6whQB823/JEdAGQnjIm7D01qf3WVgAP +a78IciejVm9hCRxUq18i7UQcmey0FcuQ4+ziFw6UOEILHN/NzR/3jiyRqOeNYDev +PfomSpT9YFF5kP2FRgjaoV4LrVUHPf066yTjztmqULSZClRKzdW4r96Tj6qiL2Ib +ZVigRwQ2mWf8RMKC9mS64eCIf4VgI1elYgW9/BRCYpV01ERTNRoIFfwm48MH7ARR +MKG92dAS70U8uV0x9jaaKYNhheLk+QLEbeJSdKF4NVY2Bd5qOX/ooZJA2wj1qvqB +tWbeXCjUSJy89FPJNCZygyyIGZT8LTwyqCNmVOrEGDjv+rzY2D8w357R9OyJydT0 +QUwd34K+tMTnlh017If8XoCaznPdruazpMCTaDi0EWjATGircWg0QiAeOVwW8VMe +BmtjGKMCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA MDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlm -aWNhdGUwHQYDVR0OBBYEFBFYhgtW76vGDcp6aW80KokkSmJgMHIGA1UdIwRrMGmA -FHrBDQizpi5ywsU3B1ncA1fWOuCyoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE +aWNhdGUwHQYDVR0OBBYEFEnxgc3u39FUWRrVmLdS1+fhF4BXMHIGA1UdIwRrMGmA +FPlf67guQYEQP2x5NI1iADjS1clRoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTALBgNVBAMM BHJvb3SCAQEwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoG -A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAnwfk -vd9EAll9Bo8G2Hj5ibLDnS4hdFnT3Z8J3Okgj8jme6vKInJrNm7wepWZEqq/Sqhs -B4uYCHYN86WrOalSqON3KsT0cue6gRKXSe/zjhNc9Uk2emJsR+uPNrUrfrIlL7Z8 -IQmewQTkfbsKL+AqT3VzIdPENnEK1N9u5w0ie5vxVz2xPbtVjqmsNgiFaAP3r8pA -mIKW3Pbay8q3wMEAWXw6EHTKelj9/7QpshzgR3AM9sYnbcr+pycRZTBEwuCv5Kvm -jbUFxY8BMZPS/b0ZbPgs8rBxE0BhMt+VlgpDbskeT25/0V2pPbAcyHQth6Qker5w -DcycKMaGGQYivLfyz+9kktIOm3AGoUdCHRxAc7xmSkIhFhQVzEgAnlK0QeojntOg -c65vbgy9x6WdTkSwMBRa4d9oNPSreTIq1ygUKOAwxZo+X38q4Oo4MTQrXDNOhGbR -/ylw/avihYJszGl9lCni6kIuqCw8CN2pvQTQcvU9UJTgy6LC7CzPqFfPjBXxN65P -WV+7N0dZ9MWHb+eAoVA/ykUrp52oStAqCYLRwCxC5NjDRXR6AdXtXTC9a3T3JLkR -LW7j8CqFmbMjhuys3Y8MEysgxKAg1PFggkeN5O5uOJXWqlUF8X2ipoMSnSIIa+Q6 -4C4CAspB8k7S5LNUVzw7V4RVLG4Mi1cSCy+Sggc= +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEAQ3yk +Hll/c8HGGi4EeA0bvIUAFRfgfBkiAyvCmm/jxarC3V4WjeXaK6qYyG7ljktT3ie6 +J+qr6WnhOveaPWJ0pvBemnn4WiiUkSRKk/jEocpbksPoA1kuaPJQxwzGl7IknkhR +tNpfgCr4XrADf7s/eYv0yqZA1ZDGrAbkJ08kfpj4HnC7Xxj00G49csFhVQq/VFAk +Y5AEiUELYNmCyq2S4yvtmsfiW6dBoxZKkZDYtfmYBM55Wmsm/vRwXW6cj+B87GY0 +w2L3CL0/496IgCRRwnDQSVANLhTJ5XhSUKcNEhEWDyXM6AIrlsJEGYYzdENqzWFh +PJDoBKnv/AKQ0nOxBOo7aOZC7WwQ3Yh/dCxDsspovUnUZRC0hD7B+kZLfggIBuK7 +F1k/SXzSKw6PNvtSAbl1V6b0LMC41TzcqoHOQuxoYYiuPoR4EwA1uOzuay6ouDSL +HHPvu1irNMtiz0PsTRHvY8G3kNujLM3MfJrxEYvE0CZvqCMVfBlxFMNexE90g11s +xdIH8Fwe5hc2DuWTrYDVcB8ZVoM7VUqgrp1KHoKfI+TbIIg6mLIthMgwpoFFbBAD +YBP0hk4bOVoRnfpi77vm+1Sc9q49JGmV7h1Zrp8b0RqmE5cgLAf+mr4YxN72Av21 +MhHKbNERB8rHVaOToLOXfE5M5Vli9ffoa6Q+E0s= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt index 7329e46a571..c541934ccaf 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt @@ -1,36 +1,36 @@ -----BEGIN CERTIFICATE----- -MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJQVDEL +MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBVMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxGDAW -BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yMTEwMTkxODI3MzFaFw0yNDA3MTUx -ODI3MzFaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz +BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yNDA3MTYxNTM1MDRaFw0yNzA0MTIx +NTM1MDRaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz Ym9uMQ4wDAYDVQQKDAVNeUxhYjEPMA0GA1UEAwwGc2VydmVyMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEAvkyxB/z2frDqa1F3lDqkW5L0F5L0sGZiOnzR -0kgPsPtxIjt3ntm+8SZAL7dJ4VSmH7DNo/IUtwent/ypN9b7rR11kl3EDcCoISPf -2VXqvi1cQou/M/GSZPDmZMsXW6kVWzZydOT4ub7sEbVtp+J4GhRkR/AZDLioBCme -Dw/GMg68+BHguPZL8c5oi+6YkhgLBvua2kum5vYMWkgATGneSZj6nWxbNRryFgq0 -X5qRrB7n7HQYjT/ki412EjbnfSpqfhcOAtu0RUI9XNx9QPx/s6PIxtcd3lgRfeB+ -tdPnDp4pQZGm6ySUSjZV+38+tKKCX+PF/u0gJWpQwA2x16lvIafxDNFd1DIr2bay -RC3vmmPMu58+B1hj0PILwnZcozBFMo/oii6m9lURUYuCBcHAKOyeMnroIXHEQMAD -aj9fFBGNAoB3gLT1gvHrrbkRmQ4b7pE3jDYR7kn4VUUxjIrfX5ucthpSIAqYFiRr -OmKUtTpwhPcRogOP36VoVM2bK4ShevhQdRZ5vjo2DggweIfIg0UBdLSethvct/hJ -fIrmq4JgUMwfHfW6HYR3Vn5T6Eag6+mwhlMXbg18D8r3XmzuAtMEruPJKdV3b4Ad -oZmnuYGvGBqHmY2fBMwdtnd7SJ3H3m+MPxF6/1Fks0D2JgVNGi9///nR6x4ITvjQ -joDODBkCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA +9w0BAQEFAAOCAg8AMIICCgKCAgEAsPeBqLtAWvBZmAtNciN1enkIgE7ffPe/8Qkz +ZK1so6LdrkQsE7v0HG8j86AdEaBTSiBfwG1+Rr5ZGpHwrPlf7Zl4GHb+v+gE3gkT +wponMN3wC8XAVExu0L7JnQBK50YEEI6/EfayQ9b80E6j2/FiyVj3rxt8MTwCAGdh +Fjb8Z8WSxLDelk9/2njOj3Swojar2QR+Yxs0mr7obOxB+ZonDSBJ21HK5xI5rOwa +wMSzyoHmW8Og7Z7TrGLei3ejvVy1RZ+pF6whQB823/JEdAGQnjIm7D01qf3WVgAP +a78IciejVm9hCRxUq18i7UQcmey0FcuQ4+ziFw6UOEILHN/NzR/3jiyRqOeNYDev +PfomSpT9YFF5kP2FRgjaoV4LrVUHPf066yTjztmqULSZClRKzdW4r96Tj6qiL2Ib +ZVigRwQ2mWf8RMKC9mS64eCIf4VgI1elYgW9/BRCYpV01ERTNRoIFfwm48MH7ARR +MKG92dAS70U8uV0x9jaaKYNhheLk+QLEbeJSdKF4NVY2Bd5qOX/ooZJA2wj1qvqB +tWbeXCjUSJy89FPJNCZygyyIGZT8LTwyqCNmVOrEGDjv+rzY2D8w357R9OyJydT0 +QUwd34K+tMTnlh017If8XoCaznPdruazpMCTaDi0EWjATGircWg0QiAeOVwW8VMe +BmtjGKMCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA MDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlm -aWNhdGUwHQYDVR0OBBYEFBFYhgtW76vGDcp6aW80KokkSmJgMHIGA1UdIwRrMGmA -FHrBDQizpi5ywsU3B1ncA1fWOuCyoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE +aWNhdGUwHQYDVR0OBBYEFEnxgc3u39FUWRrVmLdS1+fhF4BXMHIGA1UdIwRrMGmA +FPlf67guQYEQP2x5NI1iADjS1clRoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTALBgNVBAMM BHJvb3SCAQEwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoG -A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAnwfk -vd9EAll9Bo8G2Hj5ibLDnS4hdFnT3Z8J3Okgj8jme6vKInJrNm7wepWZEqq/Sqhs -B4uYCHYN86WrOalSqON3KsT0cue6gRKXSe/zjhNc9Uk2emJsR+uPNrUrfrIlL7Z8 -IQmewQTkfbsKL+AqT3VzIdPENnEK1N9u5w0ie5vxVz2xPbtVjqmsNgiFaAP3r8pA -mIKW3Pbay8q3wMEAWXw6EHTKelj9/7QpshzgR3AM9sYnbcr+pycRZTBEwuCv5Kvm -jbUFxY8BMZPS/b0ZbPgs8rBxE0BhMt+VlgpDbskeT25/0V2pPbAcyHQth6Qker5w -DcycKMaGGQYivLfyz+9kktIOm3AGoUdCHRxAc7xmSkIhFhQVzEgAnlK0QeojntOg -c65vbgy9x6WdTkSwMBRa4d9oNPSreTIq1ygUKOAwxZo+X38q4Oo4MTQrXDNOhGbR -/ylw/avihYJszGl9lCni6kIuqCw8CN2pvQTQcvU9UJTgy6LC7CzPqFfPjBXxN65P -WV+7N0dZ9MWHb+eAoVA/ykUrp52oStAqCYLRwCxC5NjDRXR6AdXtXTC9a3T3JLkR -LW7j8CqFmbMjhuys3Y8MEysgxKAg1PFggkeN5O5uOJXWqlUF8X2ipoMSnSIIa+Q6 -4C4CAspB8k7S5LNUVzw7V4RVLG4Mi1cSCy+Sggc= +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEAQ3yk +Hll/c8HGGi4EeA0bvIUAFRfgfBkiAyvCmm/jxarC3V4WjeXaK6qYyG7ljktT3ie6 +J+qr6WnhOveaPWJ0pvBemnn4WiiUkSRKk/jEocpbksPoA1kuaPJQxwzGl7IknkhR +tNpfgCr4XrADf7s/eYv0yqZA1ZDGrAbkJ08kfpj4HnC7Xxj00G49csFhVQq/VFAk +Y5AEiUELYNmCyq2S4yvtmsfiW6dBoxZKkZDYtfmYBM55Wmsm/vRwXW6cj+B87GY0 +w2L3CL0/496IgCRRwnDQSVANLhTJ5XhSUKcNEhEWDyXM6AIrlsJEGYYzdENqzWFh +PJDoBKnv/AKQ0nOxBOo7aOZC7WwQ3Yh/dCxDsspovUnUZRC0hD7B+kZLfggIBuK7 +F1k/SXzSKw6PNvtSAbl1V6b0LMC41TzcqoHOQuxoYYiuPoR4EwA1uOzuay6ouDSL +HHPvu1irNMtiz0PsTRHvY8G3kNujLM3MfJrxEYvE0CZvqCMVfBlxFMNexE90g11s +xdIH8Fwe5hc2DuWTrYDVcB8ZVoM7VUqgrp1KHoKfI+TbIIg6mLIthMgwpoFFbBAD +YBP0hk4bOVoRnfpi77vm+1Sc9q49JGmV7h1Zrp8b0RqmE5cgLAf+mr4YxN72Av21 +MhHKbNERB8rHVaOToLOXfE5M5Vli9ffoa6Q+E0s= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.jks b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.jks index 86a0a72c5db..bbf256203ec 100644 Binary files a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.jks and b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.jks differ diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.key b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.key index 8b2c6c99b72..2ca229ca24a 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.key +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.key @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJJwIBAAKCAgEAvkyxB/z2frDqa1F3lDqkW5L0F5L0sGZiOnzR0kgPsPtxIjt3 -ntm+8SZAL7dJ4VSmH7DNo/IUtwent/ypN9b7rR11kl3EDcCoISPf2VXqvi1cQou/ -M/GSZPDmZMsXW6kVWzZydOT4ub7sEbVtp+J4GhRkR/AZDLioBCmeDw/GMg68+BHg -uPZL8c5oi+6YkhgLBvua2kum5vYMWkgATGneSZj6nWxbNRryFgq0X5qRrB7n7HQY -jT/ki412EjbnfSpqfhcOAtu0RUI9XNx9QPx/s6PIxtcd3lgRfeB+tdPnDp4pQZGm -6ySUSjZV+38+tKKCX+PF/u0gJWpQwA2x16lvIafxDNFd1DIr2bayRC3vmmPMu58+ -B1hj0PILwnZcozBFMo/oii6m9lURUYuCBcHAKOyeMnroIXHEQMADaj9fFBGNAoB3 -gLT1gvHrrbkRmQ4b7pE3jDYR7kn4VUUxjIrfX5ucthpSIAqYFiRrOmKUtTpwhPcR -ogOP36VoVM2bK4ShevhQdRZ5vjo2DggweIfIg0UBdLSethvct/hJfIrmq4JgUMwf -HfW6HYR3Vn5T6Eag6+mwhlMXbg18D8r3XmzuAtMEruPJKdV3b4AdoZmnuYGvGBqH -mY2fBMwdtnd7SJ3H3m+MPxF6/1Fks0D2JgVNGi9///nR6x4ITvjQjoDODBkCAwEA -AQKCAgB6w/EFLT07+BgSycix+K7lYMVVm/l8/1/pSXL3DIQK8ftHUUPXV7Pl0H5i -CgWAd2KAWUggtuCEP00E1du46VkNkk6x8Qm0sZAFaSZVCCdbLtoxGARuly6RLhQD -/D0CZe8gG9z06m5HLhMPTZMDVoeSk4sqGYufeFTz8oFESDHdQ0LHaokl6caOpVEu -zOrifN5IT/LTVzDCGQNZDtAae8XqhuVOv4OLm+mHLLq2XnoRpxKJqU/Zd9mR6d28 -3Gy0g7OVynvAfkoq0BPUH24yiXI8yrGRDV/4e6o1Iam81YpfFMuawEMFuWBtC+am -S62BTMF86O1ni0RMvux/KZaMgYLt/tn4oUP8szwfHO4Lmgq4Rjb4hg9jvbMbZzld -1mMzCCd8y9SyEgO3E+ruAZVnQm9rfHl6rOIOp779y+5zEqS+eZqJqEy0TkBGYbeE -UGgeO/pvUBefMp9TDFY+kSOIibsK43hzI/wk6J9zFx7607RPjK4BgHMUfT4Z5ES1 -L8PIs7D1DbGUVIdpgLhQ0hjcdK4jlsNQxlnkhpEAfXlpWsxg/u7xbe/0fzAEXiiU -BP3alApjKj107Oxv8Qo0yxQt31adJLDsAhHZuGr3Hz6OMsUaSg33Rs1nIsm7ZXTT -KfH49eS2fglkzHdNU+fIf/Pog1BH6Dg+b8hqSrWaVt+FT1EaAQKCAQEA7fjz3Rp+ -EK8WrKtybGIAjYW7MbOYAWFqfQI5l4gbs3fsz9mY5+ZX4d83CZd6YAvTiF5GoYvM -G3Ey0Gi4UvtwrMAAUzIvKAxM4hg+ax1qqbdvj1OaURYrraYxqfvd5tn+aZF/6VAo -mJF/bomL7dLs8tb/Fcd+p2i4ptNUhfoqjNG0yHv066mNN6P7j0zplvP+pFBmNXD6 -YmcuhkqxcSc6X51a5UjoLqiE8bHrF8RybrTyHJCEUK2f4GeLZbKPt6gyc4YMHD4/ -r0cek14RN/WkbYSHc4Ck/RU9UF+y5aRuV3hB2M6R7KNLTVgJAp7VCRBsTjgXDwR5 -7A25X4O21j8h2QKCAQEAzLc1gheL+go9wwx9XFj/xH4Gcka8KMlLdLMWOevvdkut -YUCD94kz8DzgOY/00ZEU7Ktd21SGcc9u62s0m7UNo5XPNOqDjZLSCHOW+jjzQJK2 -WNdAeAtj41UR/vi1y8+hsL4xuewGGyPg6UavfMLHpmfgddb/DuZEFimmIsvjfUQn -MkQ0FEpHTcZldbMoQzeWrzGn5YXElhIHRxompSRWgQIctw9zmTR2pWDpR+w7WLlq -SA/y7h1ckzcau9yZ1q/5P6OLAe0LIqFamwHuKnEZfGI29f4Y4zIuIgYTtSthl8uZ -D24gZVuxENZFNgyebXeRp5BGgNvaR34yJfu1zr2UQQKCAQBoqrvPvuw9uFTXw1zD -cONfW7d258/uo4Mr0oO0M2hDqnadAM7xCG4U6ZGShW6MugbjDtwwB4XReBtqcPTJ -gtAPGy/JZ+Q30BksxI8Hm9ujJAxMVPAlxAOYD+X+Jsp10+HDWvyQiPIiELSk0Jic -FM6lDI0LbkA1r6ivr3mYp+x5HW9PUr3pvRxgoZ5ua9y9yTlx2e7qiHOIPmvHIyki -ZKeXEGW/PS7p/rLif2ewf3X5XVqz15hqx8Lu93vnUf9GPxNZnlbLQVHPbQ90r8YQ -/9i/VrAgzC9FJUdfh4fu20Cw9jK4AjtKgpQHQ7ZK0cNPhkqsqJzzvPtHKM4kxZDw -cLoZAoIBAErdaV0U/WTIoAOXqrS9+3NBX1wlUCReURDxeVxfUyhgzpw1Os2oyGy/ -5oRGiBKnyw4uMVnHlI9Ee9zuTMPwt3cujyNb7BY3LkZW90y0zQi9XqAov/PmvnXh -o34qOFm7qfb35zmF/kmomlTVO9IVbWwGqwiNQqORRC8StiYdiClEZk00shMFgwQH -XQ1pIn1Kc/nH4A3JGLaFC1JOVvmny1bMjfHIXc8WZ5ldUpKx6pYs4JhP9NhttnBE -A6vWJhBkNHO4fFqM9LiKZ5cuYon06IwbYtwdO77CmCprLIPhxIP9uhDxhjlrCdsN -cSwt+zYvNv4ffhkbzby9gttvLPLZU8ECggEADdR3MJWLFNUqruING9tJajHn2baG -ZeSc98AkGRrCYFBcyPviCg46MWfKuwhx5DMibMu3Euqiaw7BcgGgsm4BshmuSoLW -vP2iB559thv03112RD03kbUGskgA2AHqpSWh+ecnArEZRkEzir7pqebdbV0LU2MK -Z9jW8wbyGSSN1jL3zcAnbt/Qgv4nwlXqKol4P0yFvKY+5hGZzdry8Tu5n+FmwBtW -0KPbH6YFgFr4xRKzdYFX0Zt8iO2CHvxpljMLMmQVcPNycOMBormpzItjWzb97hGd -4y2v71fAR8Ms1zUb5nBa6a5o5C9d5Kj/5NAWLOcGAps5dMlmeQ1uRu0RuA== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCw94Gou0Ba8FmY +C01yI3V6eQiATt9897/xCTNkrWyjot2uRCwTu/QcbyPzoB0RoFNKIF/AbX5Gvlka +kfCs+V/tmXgYdv6/6ATeCRPCmicw3fALxcBUTG7QvsmdAErnRgQQjr8R9rJD1vzQ +TqPb8WLJWPevG3wxPAIAZ2EWNvxnxZLEsN6WT3/aeM6PdLCiNqvZBH5jGzSavuhs +7EH5micNIEnbUcrnEjms7BrAxLPKgeZbw6DtntOsYt6Ld6O9XLVFn6kXrCFAHzbf +8kR0AZCeMibsPTWp/dZWAA9rvwhyJ6NWb2EJHFSrXyLtRByZ7LQVy5Dj7OIXDpQ4 +Qgsc383NH/eOLJGo541gN689+iZKlP1gUXmQ/YVGCNqhXgutVQc9/TrrJOPO2apQ +tJkKVErN1biv3pOPqqIvYhtlWKBHBDaZZ/xEwoL2ZLrh4Ih/hWAjV6ViBb38FEJi +lXTURFM1GggV/CbjwwfsBFEwob3Z0BLvRTy5XTH2Npopg2GF4uT5AsRt4lJ0oXg1 +VjYF3mo5f+ihkkDbCPWq+oG1Zt5cKNRInLz0U8k0JnKDLIgZlPwtPDKoI2ZU6sQY +OO/6vNjYPzDfntH07InJ1PRBTB3fgr60xOeWHTXsh/xegJrOc92u5rOkwJNoOLQR +aMBMaKtxaDRCIB45XBbxUx4Ga2MYowIDAQABAoICADoBFk2g+qoUB7nPUpYryzDW +hRPRRm2moxI1StQgB/DmzDR3gd8CBzRetXEANSb23vX4AIqJGQ/FCl2sIjAAGRI+ +8F9MeZmbSiUg4ooxjtD0nd6t9alr2BKxxlGgq+x8RY7LyxLvQmT8DaeVbx4DU7bd +Y2nQ33cfzSQmvmrSoQ6+cqnnsz2fVy/b0xCwVckFlHQwheCkJSznkDVLhyByB4TE +5ZBukcwFbEQiq24tyySASWogiBn6qjThv0IFd/T0czIb8AejUs66rHjg5tEDRkyj +8aATPAi4+8z6fhN57SHFnkMRhMPAtnvpVgBhB+Ln8+sE6zvaBA08qRZ7hzrynHin +UtGC1hs1VA2WksgSVmG4IAD2TOJlUb0X3UANOZCuKlBSJmQqjw8kWX1eGoxQVpE3 +zUu4UnsZxo/vfm9bfxm06j7jKohnoY/P6mfuiV8oVuZZrJXQ7QsubYH1VT7+YtGD +xDe619hfN+kMGgkOJnc5ppE5XqUEd8C5DVkvpOy6ij8tLnQ3hVhryCWTvbay/i+3 +yLXhvAPIOy6lD7VeHKqGUrYS18MnAxFnEKQiGmVEEL6sKy1TWcZdDu+ONdCtnCEj ++RoxflPVXCGrFM/JjJfk4GkbpJ21M4i5DgCWQhZmva/SApXNhEKSOhpVQSx0WeRv +AfW+kEATT8NFpS7GnIKBAoIBAQD5J33FfqMH5IQlKZ0fz13Bd8O0DiB+ywQ+EQp3 +/p6MtZTAGI6uFfnudjMvhPlHqrVu2DsiFOQF1XUnyytm45FD/6fyXkjBCIZYh4nF +T5t2bhxhsgM0RAR9KDqfJNY6ictgq2ON2VTcTg2k4JiCTBUk8nmRpZWRtXCqB/ny +H7KdBtnQUGURGKFATTFWpZ2wpe3tFKv3P13EXheci7tStC4aPMlz69GaBvlie7ei +Y1bJ+WxGWi0c+fJo9Fi0jlc26FxsnAg3uUCdyz52oCvJnC1C7H9oHh3Gmu8EVVvm +p+r/7BAjBSMjSZ/kGApjAfRhUhheFx34E1VNKvGI9zzb1bbjAoIBAQC11ELRW/Ui +5SOwaaNp6pIIWwR/CCiF1CwzzOpZ7fRE7bQBt4pxQnvdw9gFku93liAq7POE8eVV +Vnk0h2sIh6aafwvmRLEN+upBim1eK9Isc9YdrzVTy8qer3dJeGyvTKNKY7LV2OtC +3m2rNdnYfkbzBAOgvQt4SwOaWMnKvvFY4FqTsU889yLR9eISC3Akprgqjnv7ZVcI +nJgquSgeRTmd+v3Dsuam80Phe7JfdfBG0lYGizmy7ZkLKECL1lw0KerJCFgLjUoX +L3BKEyvBZxdT+FIhc/g9OuyVmnoqp7Fs3SCV3a2TrpRsiCRxR/Vz5saD0pmlF9BQ +8mFILHqZCQNBAoIBAA6ziNVNaps9nZp/wRiuaK7QvUhziPFSU1bCNKbt7DcLQ9Kt +4EVfYJb0fIKySu8yoHAA7ERPjUDW3onRkCcLCHhFrog7jflt4ld47+UnlFnBl4Ug +NtYvqUAlTogzaTydAtYaZs5o43v4Uw/GfNk7WggLEqdIaPHnDdXKkLH7stNXIyUK +iVwf4uvjYiTlss9IiyhxDtCQ+ZxrnOMMz6nKnenIam6fnIeFH9PyztwTFnzmbcAc +9AqAqRAOUumceILx5Po3nriDAtAMvQvM2aiUwYrwX3gqRsGARNgxnTGX+OLO8AjQ +9cjQQFH6JSrAY8YWTP/IR2lq4p6iCQx24i1qh9cCggEAQhiBnXcO+ftPaN5Nr7+8 +C9uCG+dMS7Gfy8iuW7ha9f9uUr10Ts17F1cMnKIX+GGTGed+chI4ajf/4S9kofol +fr22BqNrtQhDxUVbkkLdqJkG+U1qViabZemktAlZgLOcG/t307KBepI+TWFK6h6l +DcukwfZWSAkEq0OhPtkABG2xn0Rx6FhxyN7XqfWMc1ZDVfzrZqwk4hWCXu1lfvNt +CVdmjY/ntkJxp3J+maLWj/f/m2taswcZfevI4swuQiwGk8M7th9LEniFGfg0DJ67 +5SM8P7rXPvK/c3Zu+DhzD36mTLQDV867V2R50s/0kogOG3cfsNqiN/qTkFfSzPRw +wQKCAQEAkQ1UGWeJmn8Ed9xQ1GV+NI33UIXkccBzXPxebA667zpeLcu9aXIeb4hd +8/JZedRi/C+zSoAFlIhRLtZ1vWxLq9aoJlGzuYhk2cLOz0v4edOcNwl/VWXnr8+D +g/pRq23nj8YgAj85HmOazm0cGBE8+udteNSMpVgru7Qwdk1EUPVT8OiLSk9SnOJv +V4SOjhCCA1U/luwPcDQZaGCu2cVQDQjtkAiZ9siIyy+I5QS03tgScWEPdHnJyFDW +vnq+q86pztSZtPOmbb6rQ0hM/XOkr5cu3Lncvh/2Tdko5Np7/JDBuuPVbEvzu8rt +yaeC+CDFhS18hgKDHCFUa3T5+zFfpg== +-----END PRIVATE KEY----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.p12 b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.p12 index 3181caeb3bd..34c6c99012a 100644 Binary files a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.p12 and b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.p12 differ diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt index 6ba9acb7a07..c2094a7b3a9 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt @@ -1,36 +1,36 @@ -----BEGIN CERTIFICATE----- -MIIGRjCCBC6gAwIBAgIBAzANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIGUjCCBDqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzMyWhcNMjQwNzE1MTgyNzMyWjBMMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTA0WhcNMjcwNDEyMTUzNTA0WjBMMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxDzANBgNVBAMMBnNlcnZlcjCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBAKJZa8jP+LalBAlQ8uWWlTm8TLHWkGLFxr+rNzs23mAAfLo8FFGJ -bHQ2iIWQtjCQwHiWUrHU2eRBvL5zJ3cEWte2WF/OAkNhcZu8/JcH3jCMhbjfc3V1 -xIfRdWYEhTI+5PYaqgStNs9H24fyshMFcDubXU1pKfNXtY1sQVKfQl5XeAN5Qcif -2QSfUxNOWnvnmUdBPNOIUP3W+3pqzv+uhPt9Egj2crYPXnSnC5JZSdeya+Rz60xi -WpmSWn5/WdO6j89CVI1kYX7UXuWVhtp8QLa315cMZ9+b238SrDjzNQ+ElwQJO1Nb -oD+JfsNBJBXPy/6Tr6FOZXHOWFFvIErceImKIdG12WL6IHxPlyPpeBGTdbGCS5gy -vWKaJ8diV15Mea8rGH7D30sR2StgBtW1MJOomkarS+2oQpC78HqQrNK1Qgl3GauW -PMIKCU6TR5OPZHsb87u7iXlc+NO+tBoFtgOVK7LvN/QwQ1At4hQaLcrpEJwtjfk/ -8jrC22Ewgp1LMakAsmDjGkkOBZPL5VeNgNnUEp9/AHYfH1XmpkLUOoRmUVkVDqfs -FVcFTLLb2XDtZWtl+gJ5cXxnr87dj5X2Xowo2BMKBwwMpThmJX+i2yY420hq9uu6 -xzPjUC7NauaPnIYCIjWtaBqruXWMxX9DISED71aAVAOSqXbv2fFpKMpRAgMBAAGj -ggEzMIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC +ADCCAgoCggIBAJT0RJKyh+dR+joDaEKF++Rd8YPKmo1HllITKIak0gmgPkWb+si6 +yfi7UrnmgIeSj2CESOaPcnMzbPSajBPCtitS1YlYgB8N4OKht80ib/Bc2fYblmED ++Z9+ySnVlPVFiAglS2DsVz3QUtRmpyOYUeqvBNpSZnfnxF/InIRxVAjQ/7FZ87NP +Xa42K8J1d5WXrkE/X8j7BfZUUxbKyGDB9ys8jMoHxmnH2P/P/LcvwZPaQQkxMzqk +tHY/p+tMOdkARFmAqJ1GC+zRQExRo+ppPBMeLWvQXxtW/57giqvwFoYXdFw+GHzU +0LSK810WsBDqIJfZXcd2q/EuW5mwWt6F5c5rFGI+tzYsuE+2C3sYWK7BF+BiHs1H +mZREG6+NCCTgfXaNakLFf/alouJyfj6irZXMVnFZvFmTLwJVai86l7cUYnV1A22a +O6pweBov7od5p1UmFDbD1etVa0u0HiYbdOE5rKxM0p4CDbO7FlbUoLdlE4li12sj +iCTR3psD3TG85cOiTSv1lQGpaa1MJU8a/jH3Ms24v3uFR3U/TAy++c3p2mBqfNyI +sDyIUCBEWAY/Y9YJxiS8VlN1A5WcKSkmiou2SbCX0EUIkaDoDaOiWFJY0QmVv8nM +/Keh2tPWS+NFoBLyU0iEr+kYhRloSJ3uD1opSYURtWYd3wEU9ZJVyDEvAgMBAAGj +ggE/MIIBOzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC AQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1Ud -DgQWBBQGuAQ6NLkmWyZGLYEAYVc3+43eIDB6BgNVHSMEczBxgBSWO1UVkFl5LOup -lY08xBLcfGyc+qFOpEwwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD -VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290ggkAzDBe -dcABQHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoGA1Ud -EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAoUitYEBW -lzBMuuDMHvVrtwTcKNXeOXZ6PC78OXNxGCSK4CxTzOceyHjj0K30r2qOd4hNbq43 -J60Zf+69qIpkezG3hcCy/8Yu1lLyhxLvErCbgkvZ1MH4HkEyJOeoh0PJoIq1U3vu -SA2Ny6LDA5QOWaVME6c1L0ntueu5/BddrTGtqHvK4Td9E3Y8gQhtJHCkN2GbkRRS -vr2+efC2hNIWEfxFTP+RRKKGfn+B+YASPgLDvHpIapz65MVebJH+gIo/k57swPMJ -K3Zet53tXqUvnbh87vyGdaOpq897WISNfleQ/XJB28Q3G5qBdLiTu7oxYG42XhC1 -qIm4pZ/gGW5lauu3h9bvzODB2DV6ShnBujtzbd0vBBX38BMP2oOOQ3X2QgalGOKO -jd6yTCN6VO0ZiFD4xO2XsE1CKkRqPx9rRCKR85NCvlpyytC4KIZjcomf0miMl9e1 -G9Js2y9agiXgQ5lrlZSQ8XyvRD8PC1anCxeyn8HaBblYNwUZ8S6vXkq7K13MuPgX -3vw106gUcWjw6GwC3BOi7UNtcAYyoXn6jLJ2unpn2mqjpCfbQZ3mnAI7MbEoaSFa -1CTYh9bxYK6cN+8dOnYoXUyD27aDq6xfP9YJoNmQdYTW4Wwf5JCK23j2DtDB/qen -uyydJBuI6Me+ctWntImvjM97Teu8s+ssQaA= +DgQWBBRFf07E369WKOuyKog6EyuNDDWDnDCBhQYDVR0jBH4wfIAUAdlczvOoJQH9 +cG+fm41vGD8WZaWhTqRMMEoxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0G +A1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVNeUxhYjENMAsGA1UEAwwEcm9vdIIUM0vS +1bRQQCJzcfsSQBT78uhQ7lUwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG +AQUFBwMBMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAgEAPW9KkG+G/oRmCiB23nxOJz7R6w0S2eM938RaKkRhtXdXBmVIusr1vdFm +QvjMA3RgoQVTyFwD2PIvpFmFAYThZrxu8a9TIXqVrgHLvHgRuWVlXbkq/fsi/AEH +BNbJS7WoO1t+QxvOVXota/hUYMhes8QvRTDqql6iXdV6/mzLMxJ7vpRmnV6zaRmo +cMegAH82IEeXz61N++Cn+Tzasy2mkH/eZ+pXaPrCKhwWD08zEDSm9ury2LJzPWCT +GbHkwgJy8TwIRLCpORlccIoZED0wyeQNAuMRS3vh5w2UpHz1CAot0dF2XzPwkJaW +eJTHmU9iRmPnn9VgCSMYLadjANT1SGFeLJlYxAHmbheMKepkFddiqyCUVjIOyN79 +7Uzgl4iPSFrHSyNgeXuIEEmNb9jQlcxPzNcyjz+XPYAeh9RihvZgi0TalPJqFc57 +HKqDoUnHfb2iyN2wmMca9xKhBgQycFxsoDr+GlWehkPMf0ZIqR757v65/0JRGQuf +2VMyTYm3CsRgIuhrI2KMCeDrOeHrRAz/A9Sa+PI7Arvu0lPaTcVCAz7Nw9DnAOhR +kfC1mK+wqPYHKZduYGyBmZsGylAOaiO9KGWyzk11qI9Bk4nmkJ8KS/0rmXYAwZB3 +dbEceSt9xvY3oY0R+qZFUEF5ztHaBiggScS9duCD60iZqwScAPY= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt index 6ba9acb7a07..c2094a7b3a9 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt @@ -1,36 +1,36 @@ -----BEGIN CERTIFICATE----- -MIIGRjCCBC6gAwIBAgIBAzANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIGUjCCBDqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzMyWhcNMjQwNzE1MTgyNzMyWjBMMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTA0WhcNMjcwNDEyMTUzNTA0WjBMMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxDzANBgNVBAMMBnNlcnZlcjCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBAKJZa8jP+LalBAlQ8uWWlTm8TLHWkGLFxr+rNzs23mAAfLo8FFGJ -bHQ2iIWQtjCQwHiWUrHU2eRBvL5zJ3cEWte2WF/OAkNhcZu8/JcH3jCMhbjfc3V1 -xIfRdWYEhTI+5PYaqgStNs9H24fyshMFcDubXU1pKfNXtY1sQVKfQl5XeAN5Qcif -2QSfUxNOWnvnmUdBPNOIUP3W+3pqzv+uhPt9Egj2crYPXnSnC5JZSdeya+Rz60xi -WpmSWn5/WdO6j89CVI1kYX7UXuWVhtp8QLa315cMZ9+b238SrDjzNQ+ElwQJO1Nb -oD+JfsNBJBXPy/6Tr6FOZXHOWFFvIErceImKIdG12WL6IHxPlyPpeBGTdbGCS5gy -vWKaJ8diV15Mea8rGH7D30sR2StgBtW1MJOomkarS+2oQpC78HqQrNK1Qgl3GauW -PMIKCU6TR5OPZHsb87u7iXlc+NO+tBoFtgOVK7LvN/QwQ1At4hQaLcrpEJwtjfk/ -8jrC22Ewgp1LMakAsmDjGkkOBZPL5VeNgNnUEp9/AHYfH1XmpkLUOoRmUVkVDqfs -FVcFTLLb2XDtZWtl+gJ5cXxnr87dj5X2Xowo2BMKBwwMpThmJX+i2yY420hq9uu6 -xzPjUC7NauaPnIYCIjWtaBqruXWMxX9DISED71aAVAOSqXbv2fFpKMpRAgMBAAGj -ggEzMIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC +ADCCAgoCggIBAJT0RJKyh+dR+joDaEKF++Rd8YPKmo1HllITKIak0gmgPkWb+si6 +yfi7UrnmgIeSj2CESOaPcnMzbPSajBPCtitS1YlYgB8N4OKht80ib/Bc2fYblmED ++Z9+ySnVlPVFiAglS2DsVz3QUtRmpyOYUeqvBNpSZnfnxF/InIRxVAjQ/7FZ87NP +Xa42K8J1d5WXrkE/X8j7BfZUUxbKyGDB9ys8jMoHxmnH2P/P/LcvwZPaQQkxMzqk +tHY/p+tMOdkARFmAqJ1GC+zRQExRo+ppPBMeLWvQXxtW/57giqvwFoYXdFw+GHzU +0LSK810WsBDqIJfZXcd2q/EuW5mwWt6F5c5rFGI+tzYsuE+2C3sYWK7BF+BiHs1H +mZREG6+NCCTgfXaNakLFf/alouJyfj6irZXMVnFZvFmTLwJVai86l7cUYnV1A22a +O6pweBov7od5p1UmFDbD1etVa0u0HiYbdOE5rKxM0p4CDbO7FlbUoLdlE4li12sj +iCTR3psD3TG85cOiTSv1lQGpaa1MJU8a/jH3Ms24v3uFR3U/TAy++c3p2mBqfNyI +sDyIUCBEWAY/Y9YJxiS8VlN1A5WcKSkmiou2SbCX0EUIkaDoDaOiWFJY0QmVv8nM +/Keh2tPWS+NFoBLyU0iEr+kYhRloSJ3uD1opSYURtWYd3wEU9ZJVyDEvAgMBAAGj +ggE/MIIBOzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC AQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1Ud -DgQWBBQGuAQ6NLkmWyZGLYEAYVc3+43eIDB6BgNVHSMEczBxgBSWO1UVkFl5LOup -lY08xBLcfGyc+qFOpEwwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD -VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290ggkAzDBe -dcABQHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoGA1Ud -EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAoUitYEBW -lzBMuuDMHvVrtwTcKNXeOXZ6PC78OXNxGCSK4CxTzOceyHjj0K30r2qOd4hNbq43 -J60Zf+69qIpkezG3hcCy/8Yu1lLyhxLvErCbgkvZ1MH4HkEyJOeoh0PJoIq1U3vu -SA2Ny6LDA5QOWaVME6c1L0ntueu5/BddrTGtqHvK4Td9E3Y8gQhtJHCkN2GbkRRS -vr2+efC2hNIWEfxFTP+RRKKGfn+B+YASPgLDvHpIapz65MVebJH+gIo/k57swPMJ -K3Zet53tXqUvnbh87vyGdaOpq897WISNfleQ/XJB28Q3G5qBdLiTu7oxYG42XhC1 -qIm4pZ/gGW5lauu3h9bvzODB2DV6ShnBujtzbd0vBBX38BMP2oOOQ3X2QgalGOKO -jd6yTCN6VO0ZiFD4xO2XsE1CKkRqPx9rRCKR85NCvlpyytC4KIZjcomf0miMl9e1 -G9Js2y9agiXgQ5lrlZSQ8XyvRD8PC1anCxeyn8HaBblYNwUZ8S6vXkq7K13MuPgX -3vw106gUcWjw6GwC3BOi7UNtcAYyoXn6jLJ2unpn2mqjpCfbQZ3mnAI7MbEoaSFa -1CTYh9bxYK6cN+8dOnYoXUyD27aDq6xfP9YJoNmQdYTW4Wwf5JCK23j2DtDB/qen -uyydJBuI6Me+ctWntImvjM97Teu8s+ssQaA= +DgQWBBRFf07E369WKOuyKog6EyuNDDWDnDCBhQYDVR0jBH4wfIAUAdlczvOoJQH9 +cG+fm41vGD8WZaWhTqRMMEoxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0G +A1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVNeUxhYjENMAsGA1UEAwwEcm9vdIIUM0vS +1bRQQCJzcfsSQBT78uhQ7lUwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG +AQUFBwMBMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAgEAPW9KkG+G/oRmCiB23nxOJz7R6w0S2eM938RaKkRhtXdXBmVIusr1vdFm +QvjMA3RgoQVTyFwD2PIvpFmFAYThZrxu8a9TIXqVrgHLvHgRuWVlXbkq/fsi/AEH +BNbJS7WoO1t+QxvOVXota/hUYMhes8QvRTDqql6iXdV6/mzLMxJ7vpRmnV6zaRmo +cMegAH82IEeXz61N++Cn+Tzasy2mkH/eZ+pXaPrCKhwWD08zEDSm9ury2LJzPWCT +GbHkwgJy8TwIRLCpORlccIoZED0wyeQNAuMRS3vh5w2UpHz1CAot0dF2XzPwkJaW +eJTHmU9iRmPnn9VgCSMYLadjANT1SGFeLJlYxAHmbheMKepkFddiqyCUVjIOyN79 +7Uzgl4iPSFrHSyNgeXuIEEmNb9jQlcxPzNcyjz+XPYAeh9RihvZgi0TalPJqFc57 +HKqDoUnHfb2iyN2wmMca9xKhBgQycFxsoDr+GlWehkPMf0ZIqR757v65/0JRGQuf +2VMyTYm3CsRgIuhrI2KMCeDrOeHrRAz/A9Sa+PI7Arvu0lPaTcVCAz7Nw9DnAOhR +kfC1mK+wqPYHKZduYGyBmZsGylAOaiO9KGWyzk11qI9Bk4nmkJ8KS/0rmXYAwZB3 +dbEceSt9xvY3oY0R+qZFUEF5ztHaBiggScS9duCD60iZqwScAPY= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.jks b/qa/integration/fixtures/webserver_certs/generated/server_from_root.jks index 1596eef7b5d..11414c8e6b4 100644 Binary files a/qa/integration/fixtures/webserver_certs/generated/server_from_root.jks and b/qa/integration/fixtures/webserver_certs/generated/server_from_root.jks differ diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.key b/qa/integration/fixtures/webserver_certs/generated/server_from_root.key index 64ce87d6bd9..b8bf4c8afe3 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_root.key +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_root.key @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAollryM/4tqUECVDy5ZaVObxMsdaQYsXGv6s3OzbeYAB8ujwU -UYlsdDaIhZC2MJDAeJZSsdTZ5EG8vnMndwRa17ZYX84CQ2Fxm7z8lwfeMIyFuN9z -dXXEh9F1ZgSFMj7k9hqqBK02z0fbh/KyEwVwO5tdTWkp81e1jWxBUp9CXld4A3lB -yJ/ZBJ9TE05ae+eZR0E804hQ/db7emrO/66E+30SCPZytg9edKcLkllJ17Jr5HPr -TGJamZJafn9Z07qPz0JUjWRhftRe5ZWG2nxAtrfXlwxn35vbfxKsOPM1D4SXBAk7 -U1ugP4l+w0EkFc/L/pOvoU5lcc5YUW8gStx4iYoh0bXZYvogfE+XI+l4EZN1sYJL -mDK9Yponx2JXXkx5rysYfsPfSxHZK2AG1bUwk6iaRqtL7ahCkLvwepCs0rVCCXcZ -q5Y8wgoJTpNHk49kexvzu7uJeVz40760GgW2A5Ursu839DBDUC3iFBotyukQnC2N -+T/yOsLbYTCCnUsxqQCyYOMaSQ4Fk8vlV42A2dQSn38Adh8fVeamQtQ6hGZRWRUO -p+wVVwVMstvZcO1la2X6AnlxfGevzt2PlfZejCjYEwoHDAylOGYlf6LbJjjbSGr2 -67rHM+NQLs1q5o+chgIiNa1oGqu5dYzFf0MhIQPvVoBUA5Kpdu/Z8WkoylECAwEA -AQKCAgAnC76ik3JL6zAnVXK1yRumwSTo+KPNPafpXTJxaWbeS/+zgtOYMKcG5D69 -cOqAj6/a0LyezISDI1RTsbpiI4f9rXtfv1pH/l2miTNu15OWtZVLu2I3c9WKnNPR -b3DhfsLudcdCTK6L98pSzgoKDRrLGzi7b3mJbyTZGJo1VtxNeFgC7dzmdZtqQy9l -y0hz2GXC/2uzrGteQaBF9v5F/CE7ov79bNh19QhQogRkdFkQKXdNgJusyVLk8JAX -MGXYSFxiCp/vSZ8xM76HFQlT7uJZj6moSHGLFB8cVs9Y+xVd4BO6AsZj6/uEjHsS -Vs5o+IsldTueXg6D6o7PNxzbJF9QwvCWfIPfb2Zo/cRMN1234xCbYV+7GUSulwyX -0av1L6G8ueVNd+X/8cq3lS5yD49ZkvhO4EjdR17thrgUQotYqG7JrwQsUq4AAB4o -eHa8utxpyEaUCqVJ9TZLp2267xyE81Rx9DsN6oX+m/M2C0fI9tveCO7kpn7l5kYi -WhBZGH64t3PpgFhYz1C37Ruz5vK1k1O2v4F7zKCNyTKaARx/8cStcInXVVwqe1fW -WCnzZZ7W8mtD4ni0lI+yhu4nToLHgqGPUuYy99xINaLgPuLZycCHzNGqmuj+TCPI -Hhe4hJ1RuXDXEdeG82XkjrvG4tbHpmp6kQQSVcwle1mdGqBpqQKCAQEA0XwXxsER -IHK/5wVY/KOWmlX+LnizstpsHCsD+tvzotCeUwZfhxXeyxHVxJaBMB03GWZFnbOM -i59LTsodKcjM6sWJNYd45tRQoR10zNz68QqiIxQ8KksW/WCsJHdaJhvxNG7bReen -qN4jtKFExqWICvm+zGzBYlDRaY6QRlJz4xhZpCy7228aztisJOPTkPyIgKODYlIu -O+6wRJq77j8kIO4FmnqZCwkzrPeq8q4stxVG/2Qgc6sNPobWd91nu0HPeJtyj813 -QU5Ni4OhWk/mF7pZYzfCvDazdcLSE6MDM/k5dtJcKkFdom3+MbsTPIpqDM16whts -FZPmMdOAGFr/AwKCAQEAxmX4m81hvxPTA47vNtw5NuYZv0rpB+4Cavcpfp0eyea2 -i/ZTpVOLRNGkSHnszQkUFY42YYDmrc1B5DUIBqZAp3IrYXjOjuMWpmB8/jPwp47A -kOvFlMuUMcn4R2pI+Ke/B0nttNx+dmAe1OQiq5CCCXDcdPY/7nbR1AVgGPPap1i6 -4MA6lBA/5AC1JNG/tmAwGFN/dVLaF61X301RgCvjiAA7lXxx5pw0HUXE1UCEgNuu -RXtOhUtnPy3bir+PT/cOZKneWbeNnVKxhcTB1PvC5Z1BD5V2CP+CdLrNUIFjwjCg -um+T0yXQ5qdtRpIkldAB0dbnWQaLswfvJYgCvSz3GwKCAQBWEGKAUJ+lsh8vXZ4S -pW7hV4UHOKWzTFNwOyf2b1koQgdZ2ttyV0TiLyChFKhcyRPzBnaBdNvAHSDGbPkQ -SqQNrDxT1qWQSBgcKUrsQ7mBQDoeqViSk6kNTiNLFBHgaEhvnz/M94CFO0Z2Fm94 -IVfEEpkVyXSl3Mg8UsrpaUSdP0rMzWgcngfohVIsEcHA5k96cjMvOM4v5z9Z4Cxn -lF9WfBOnRMkNpKkrXdFIxNAwI7i4gj1m6BaX4rPIKRlN8Jq5VTv64pIlLXNkZx7q -SLsicnf48Fa8FskiD3lZgfZ16uV18LTW02YRrpjpe49Kpx8sxVqy1qtQLiQ7wkX1 -O6VbAoIBAQCTYG5D9lYU5boLW6MxQngx4s3fftsKK/CZ2s51ImwbmQmjgLD0D9JV -pdF+kSupx4Om24Rj+6fgFahLyGUU420Ftdapyl7OX7yUjPpsrtKm9XDwdmPIO03z -ik+nZTmdlKZf2Lf+cQh0EMT2ah2jCm6+apspOz+Ueak28WNIReT4CP6RM9qP4F5F -lYz6M+FQrbUsbOIymgAuNHMhCsNvDaswfNPGILJPl1tTH1+u9gU333heXtGeTLv+ -f6clM1VXFnQXrOPiC2rkIeiDMPy3pOlLPfx1ghYzavJSme0oT2V2fD75HydVYzMC -E/8lvCv3KZQgd55VUHp33H32iElYk81fAoIBADZCrhiWKBAND7PAbHBIw3seOH+/ -tsedSQuzVCZN32LhsLt3Xc1wB3l2xJlY1wIVr8tFvRPnwGPz15G89uTE/EGvCyLg -QW2Ot2kRbc4ECjbeHBUsJT63L4tM4Kq17KPb9ew1V2kaLqrN0ZbBu9oFyCSEodSk -g3ft5jYF3fgBBS/04fn7ZPHKJ2isx2vnEzKqTMLy6R09jyFE2yD+lLMGO9HBnEAo -2/o3CXKWAGSWZag6wVJ6dVribV13VdcoPgcBVZW2nTvN2M742pcb9SjNghMQ6htH -2v5+jFi5E8ApvBPX7ii2E2OLBrxJC/2CaF6VIF2iQaqwAl31zmuors7eQL0= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCU9ESSsofnUfo6 +A2hChfvkXfGDypqNR5ZSEyiGpNIJoD5Fm/rIusn4u1K55oCHko9ghEjmj3JzM2z0 +mowTwrYrUtWJWIAfDeDiobfNIm/wXNn2G5ZhA/mffskp1ZT1RYgIJUtg7Fc90FLU +ZqcjmFHqrwTaUmZ358RfyJyEcVQI0P+xWfOzT12uNivCdXeVl65BP1/I+wX2VFMW +yshgwfcrPIzKB8Zpx9j/z/y3L8GT2kEJMTM6pLR2P6frTDnZAERZgKidRgvs0UBM +UaPqaTwTHi1r0F8bVv+e4Iqr8BaGF3RcPhh81NC0ivNdFrAQ6iCX2V3HdqvxLluZ +sFreheXOaxRiPrc2LLhPtgt7GFiuwRfgYh7NR5mURBuvjQgk4H12jWpCxX/2paLi +cn4+oq2VzFZxWbxZky8CVWovOpe3FGJ1dQNtmjuqcHgaL+6HeadVJhQ2w9XrVWtL +tB4mG3ThOaysTNKeAg2zuxZW1KC3ZROJYtdrI4gk0d6bA90xvOXDok0r9ZUBqWmt +TCVPGv4x9zLNuL97hUd1P0wMvvnN6dpganzciLA8iFAgRFgGP2PWCcYkvFZTdQOV +nCkpJoqLtkmwl9BFCJGg6A2jolhSWNEJlb/JzPynodrT1kvjRaAS8lNIhK/pGIUZ +aEid7g9aKUmFEbVmHd8BFPWSVcgxLwIDAQABAoICACE+Cne6xD3Ju9BqpkACat7i +YF4LP0a8bT3P7MFv0zg2vO8ICoX1udF67NF0MccP3FOr8iESbnZrSaulOhHG6KTu +jZ1jVOSZUlEBymVLhDPeZP5Z1cu8+V6xCu/9LIs4g01eFV6CIE45SeMBLdfxzIZV +nP2kWh30aYz+1n9wU1LMYeQQsSSZV8zjZgy17oRCH6+MFzJhY0lANHLnuqRtZV4O +21TTO2KsKS+eGUZAsDDxCoakpkw2KdgiCLWog1h8iMR4xCilU0/8ux7NMYnFBpU1 +rOlczXzqg43PiuMoZ75xSZS0fv6M+C7yu7FQuOZteeyppBY509G1iNfZ1c3HkCuT +2WXN8ihYwNCuJggZFhgpEua54pNrnpseQOmFDKfhsbZ503axkb/Fra7gIXRgNdpU +/MyUjIuF3SdNdlZxba/MFROulPGPTkSe/NRhS51YO0FGVzhw8y8XarOzliBChGOZ +P7MOTTBBdeVDgKhZtuVTeGlbOtgBbnBQigEmupqvP51k/AD48P/UqdEq+f+6xs+t +NwBRBXSOU75Fe0h77AHtgQhFf5dMN3UCqsdP3ZHLGjplbuBUwGKhRVZ2OlAsY8M1 +HkiuKYVS6BZIC2ZcJnid372q8DHc5lpd0uTt33S1SptoQ4efWgdGYUQ5qqkCH9rG +C7SznflFtdRpINUOVB1BAoIBAQDLoQho8FpGh0OG7hIUKLr9/bntVUMrLuzHeaeE +XPMIcZou2ouQfl8yJllwwTBHIrls1Zb4cv+TeP3wSjDaAUpEJjvhrtzUZUTe6HbR +ypWaAKtP1dBC9ViNyeMs9AcfqV/UebE8Fs+ggsejxaMSVkWBLbYwCyfePFqtRveT +RA5KswDC5/EMHFesCBc2yijSDrpIKykznwef5iwPxnwHOfccb5cDzuY7/au4sEwH +PgKb2cSargb752ByGPQbW170u1KX2BeydQlPdxIgvUdLwLnbxoi+1SkJeAmOi8r7 +4pi3+7Y2qwK1+xaZJ9QgkIkoU4gjPfXKglbQXco8VS26U/HfAoIBAQC7Q279gmq1 +VnGvXW6h/d8ff/OdDnqQmgExTJXYLqBjIgRI0tkuETdDpICkwaIN7gYV74PbMXnh +37ldtcBrpbOHpvZ5hRThtJUD1K5tcC1RGHErQ9kBbShH059gqSk6p5ZM4dTExBfH +DtAPylDuGMDAr6fv5cuvUmkEzRknFu0Dm82t1lX0iKX4T7xvr78/i1eML9aCJ73M +PYi8QRgHzMJMH2MsgHCvOgSb9X1On9BO2+vX4V7nl74EoeSMKtRDl/2O236QQL4d +Dn6sZlZlI+MLplvTfQJ7VOHGM/8knhr3gzvzuIR88CWPWkjVnlb9hLR67BxvTf+7 +BQuYyd8/nMqxAoIBABGd/wJFOSo87aSw2DOSSotK9lkINWV5C9c+OTYEbKpgWbyP +/98ib84A72AH9stQv8YgvI0GNgJmR2IQqSVTUz+OJ0QhQs3FbS3z4sPHySWnxSqF +Dx2b6Gtr+hGbkzo0Byx00tamCpSSe/85TCB8AktozwhdDrf1dMrlBupgyvO/hONX ++czYbuVdVdTRrLJvv8IaqdfjfF9jJJDA1V4yr6RwKxqTPewWyFKUPh+qR8OAFbhk +Mgp9PCTi78umHhMCWeT1NyejZIjRfmoCtjZhmDLhfFc2TfSZxAnnIkqkkh3o0xzr +srJX5N6fSYT/vD4W7AJQjAFXQXeJhcOuguRJ+vUCggEBAKa2lRQ6TBeemt1dl8W9 +xys69T4IitM7V/50yR5J/F3lBd1zYQgjKYkd5a1H+abfFinnLwMwW+rcCyHDUkhD +n74PxXRQJ1P9OYDMknE5og6FbRLwdzxO2sJ/Q2+BigP++M7/ajW2wdhkdpToL/6Y +qNmiSCdJ9HjkLDyq/06OmXlR9t1haOqn0OJs0jy4QNIokktqt1TuXol66sV2Shd/ +kibE9rzCqA9H3pyeA+3AV2otXhy2NpCFxhRHsLmc05ZNtN2+yUFRci9aHSoMfbPT +LCnN9AgnTdNQsSOMXQRTgXyDKBeFWOYFwdqKrTcInTdcMnUQyQvKa/YqurVOW2M9 +ijECggEBAIPUZtCdVZEWR+D/vxhaMBJEemf7Dylqfw0zYOERUURZIrBTuPgCf7RO +EL6AK+Ca3cnrhwf1hVm4MUAiTIrz8J0JYD4mrGjaOfSrEsDh47ftnCJ5klANjvxp +pTUENU8bru/jo2Og+ET/novW5UbeFgFt3BTgN2NXP6p3vRtvpclH5qo7TViFCHLN +nIx8m5/YMXe6wRAghexJTOLZkOZ0aqwJ3cm4IFtK1U4je6ikEOAByNldupWSgM4o +9dlF1xSydwxj4qIO5wlgrzpTjdNMsz8OL7jFzTkv6HbBY5h4OZ41kmjunTbmDt6A +rnPB2MB9LPFWnVBTU5c1QVVZkxwc7KQ= +-----END PRIVATE KEY----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.p12 b/qa/integration/fixtures/webserver_certs/generated/server_from_root.p12 index 11d41714b40..b30db573f65 100644 Binary files a/qa/integration/fixtures/webserver_certs/generated/server_from_root.p12 and b/qa/integration/fixtures/webserver_certs/generated/server_from_root.p12 differ diff --git a/qa/integration/specs/multiple_pipeline_spec.rb b/qa/integration/specs/multiple_pipeline_spec.rb index 31496336610..cb01e56fe5f 100644 --- a/qa/integration/specs/multiple_pipeline_spec.rb +++ b/qa/integration/specs/multiple_pipeline_spec.rb @@ -34,6 +34,7 @@ let(:temporary_out_file_1) { Stud::Temporary.pathname } let(:temporary_out_file_2) { Stud::Temporary.pathname } + let(:temporary_out_file_3) { Stud::Temporary.pathname } let(:pipelines) {[ { @@ -47,6 +48,12 @@ "pipeline.workers" => 1, "pipeline.batch.size" => 1, "config.string" => "input { generator { count => 1 } } output { file { path => \"#{temporary_out_file_2}\" } }" + }, + { + "pipeline.id" => "config-string-with-env-var-pipeline", + "pipeline.workers" => 1, + "pipeline.batch.size" => 1, + "config.string" => "input { generator { count => 1 } } output { file { path => \"${TEMP_FILE_PATH}\" } }" } ]} @@ -65,6 +72,7 @@ it "executes the multiple pipelines" do logstash_service = @fixture.get_service("logstash") + logstash_service.env_variables = {'TEMP_FILE_PATH' => temporary_out_file_3} logstash_service.spawn_logstash("--path.settings", settings_dir, "--log.level=debug") try(retry_attempts) do expect(logstash_service.exited?).to be(true) @@ -74,6 +82,8 @@ expect(IO.readlines(temporary_out_file_1).size).to eq(1) expect(File.exist?(temporary_out_file_2)).to be(true) expect(IO.readlines(temporary_out_file_2).size).to eq(1) + expect(File.exist?(temporary_out_file_3)).to be(true) + expect(IO.readlines(temporary_out_file_3).size).to eq(1) end context 'effectively-empty pipelines.yml file' do diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 7d5d7e4ea92..f4a8657c4e4 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -50,7 +50,6 @@ namespace "artifact" do "logstash-core-plugin-api/*.gemspec", "patterns/**/*", - "tools/ingest-converter/build/libs/ingest-converter.jar", "vendor/??*/**/*", # To include ruby-maven's hidden ".mvn" directory, we need to # do add the line below. This directory contains a file called @@ -492,6 +491,7 @@ namespace "artifact" do require "zlib" require 'rubygems' require 'rubygems/package' + require 'minitar' ensure_logstash_version_constant_defined tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}#{platform}.tar.gz" if File.exist?(tarpath) && ENV['SKIP_PREPARE'] == "1" && !source_modified_since?(File.mtime(tarpath)) @@ -500,7 +500,7 @@ namespace "artifact" do end puts("[artifact:tar] building #{tarpath}") gz = Zlib::GzipWriter.new(File.new(tarpath, "wb"), Zlib::BEST_COMPRESSION) - Gem::Package::TarWriter.new(gz) do |tar| + Minitar::Writer.open(gz) do |tar| files(exclude_paths).each do |path| write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}") end @@ -520,11 +520,11 @@ namespace "artifact" do def write_to_tar(tar, path, path_in_tar) stat = File.lstat(path) if stat.directory? - tar.mkdir(path_in_tar, stat.mode) + tar.mkdir(path_in_tar, :mode => stat.mode) elsif stat.symlink? - tar.add_symlink(path_in_tar, File.readlink(path), stat.mode) + tar.symlink(path_in_tar, File.readlink(path), :mode => stat.mode) else - tar.add_file_simple(path_in_tar, stat.mode, stat.size) do |io| + tar.add_file_simple(path_in_tar, :mode => stat.mode, :size => stat.size) do |io| File.open(path, 'rb') do |fd| chunk = nil size = 0 diff --git a/settings.gradle b/settings.gradle index 4415c1417c6..aa27de37971 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,10 +1,9 @@ rootProject.name = "logstash" -include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter', 'benchmark-cli', 'jvm-options-parser', 'logstash-integration-tests', 'dependencies-report' +include ':logstash-core', 'logstash-core-benchmarks', 'benchmark-cli', 'jvm-options-parser', 'logstash-integration-tests', 'dependencies-report' project(':logstash-core').projectDir = new File('./logstash-core') project(':logstash-core-benchmarks').projectDir = new File('./logstash-core/benchmarks') project(':logstash-integration-tests').projectDir = new File('./qa/integration') -project(':ingest-converter').projectDir = new File('./tools/ingest-converter') project(':benchmark-cli').projectDir = new File('./tools/benchmark-cli') project(':dependencies-report').projectDir = new File('./tools/dependencies-report') project(':jvm-options-parser').projectDir = new File('./tools/jvm-options-parser') diff --git a/tools/ingest-converter/build.gradle b/tools/ingest-converter/build.gradle deleted file mode 100644 index 62e4437a6ab..00000000000 --- a/tools/ingest-converter/build.gradle +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.yaml.snakeyaml.Yaml - -// fetch version from Logstash's main versions.yml file -def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../../versions.yml").text) - -description = """Ingest JSON to Logstash Grok Config Converter""" -version = versionMap['logstash-core'] -String jacksonDatabindVersion = versionMap['jackson-databind'] - -repositories { - mavenCentral() -} - -buildscript { - repositories { - mavenCentral() - gradlePluginPortal() - } - dependencies { - classpath "org.yaml:snakeyaml:${snakeYamlVersion}" - classpath "com.github.johnrengelman:shadow:${shadowGradlePluginVersion}" - } -} - -dependencies { - implementation 'net.sf.jopt-simple:jopt-simple:4.6' - implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}" - testImplementation "junit:junit:4.13.2" - testImplementation 'commons-io:commons-io:2.16.1' -} - -javadoc { - enabled = true -} - -apply plugin: 'com.github.johnrengelman.shadow' - -shadowJar { - archiveBaseName = 'ingest-converter' - archiveClassifier = null - archiveVersion = '' -} - -assemble.dependsOn shadowJar diff --git a/tools/ingest-converter/gradle.properties b/tools/ingest-converter/gradle.properties deleted file mode 100644 index 7f7a1fae05e..00000000000 --- a/tools/ingest-converter/gradle.properties +++ /dev/null @@ -1,2 +0,0 @@ -isDistributedArtifact=false - diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java deleted file mode 100644 index 131805f22da..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Append DSL to Logstash mutate Transpiler. - */ -public final class Append { - - private Append() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_append_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java deleted file mode 100644 index 802eb4e391b..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Convert DSL to Logstash Date Transpiler. - */ -public final class Convert { - - private Convert() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_convert_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java deleted file mode 100644 index 91237ff07d0..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Date DSL to Logstash Date Transpiler. - */ -public final class Date { - - private Date() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_date"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java deleted file mode 100644 index 949ca243f63..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -public final class GeoIp { - - private GeoIp() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_geoip"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java deleted file mode 100644 index 44d9f189010..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest JSON DSL to Logstash Grok Transpiler. - */ -public final class Grok { - - private Grok() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_grok"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java deleted file mode 100644 index 29ee3d01f1e..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -public final class Gsub { - - private Gsub() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_gsub"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java deleted file mode 100644 index eee5d8dfbdd..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestAppend { - - /** - * Converts Ingest Append JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestAppend::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", appendHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String appendHash(Map processor) { - Map append_json = processor.get("append"); - Object value = append_json.get("value"); - Object value_contents; - if (value instanceof List) { - value_contents = IngestConverter.createArray((List) value); - } else { - value_contents = IngestConverter.quoteString((String) value); - } - Object mutate_contents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) append_json.get("field"))), - (String) value_contents); - return IngestConverter.createField("add_field", IngestConverter.wrapInCurly((String) mutate_contents)); - } - - public static boolean has_append(Map processor) { - return processor.containsKey("append"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java deleted file mode 100644 index 6c1128264ac..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestConvert { - - /** - * Converts Ingest Convert JSON to LS Date filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestConvert::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", convertHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String convertHash(Map processor) { - Map convert_json = processor.get("convert"); - - Object mutate_contents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) convert_json.get("field"))), - IngestConverter.quoteString((String) convert_json.get("type"))); - return IngestConverter.createField("convert", IngestConverter.wrapInCurly((String) mutate_contents)); - } - - public static boolean has_convert(Map processor) { - return processor.containsKey("convert"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java deleted file mode 100644 index 16bc91e3c3f..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.logstash.ingest; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class IngestConverter { - - /** - * Translates the JSON naming pattern (`name.qualifier.sub`) into the LS pattern - * [name][qualifier][sub] for all applicable tokens in the given string. - * This function correctly identifies and omits renaming of string literals. - * @param content to replace naming pattern in - * @returns {string} with Json naming translated into grok naming - */ - public static String dotsToSquareBrackets(String content) { - final Pattern pattern = Pattern.compile("\\(\\?:%\\{.*\\|-\\)"); - final Matcher matcher = pattern.matcher(content); - List tokens = new ArrayList<>(); - String right = content; - while (matcher.find()) { - final int start = matcher.start(); - final int end = matcher.end(); - final String matchContent = content.substring(start, end); - right = content.substring(end); - tokens.add(tokenDotsToSquareBrackets(content.substring(0, start))); - tokens.add(matchContent); - } - tokens.add(tokenDotsToSquareBrackets(right)); - return String.join("", tokens); - } - - private static String tokenDotsToSquareBrackets(String content) { - //Break out if this is not a naming pattern we convert - final String adjusted; - if (Pattern.compile("([\\w_]+\\.)+[\\w_]+").matcher(content).find()) { - adjusted = content.replaceAll("(\\w*)\\.(\\w*)", "$1][$2") - .replaceAll("\\[(\\w+)(}|$)", "[$1]$2") - .replaceAll("\\{(\\w+):(\\w+)]", "{$1:[$2]") - .replaceAll("^(\\w+)]\\[", "[$1]["); - } else { - adjusted = content; - } - return adjusted; - } - - public static String quoteString(String content) { - return "\"" + content.replace("\"", "\\\"") + "\""; - } - - public static String wrapInCurly(String content) { - return "{\n" + content + "\n}"; - } - - public static String createField(String fieldName, String content) { - return fieldName + " => " + content; - } - - public static String createHash(String fieldName, String content) { - return fieldName + " " + wrapInCurly(content); - } - - /** - * All hash fields in LS start on a new line. - * @param fields Array of Strings of Serialized Hash Fields - * @returns {string} Joined Serialization of Hash Fields - */ - public static String joinHashFields(String... fields) { - return String.join("\n", fields); - } - - /** - * Fixes indentation in LS string. - * @param content LS string to fix indentation in, that has no indentation intentionally with - * all lines starting on a token without preceding spaces. - * @return LS string indented by 3 spaces per level - */ - public static String fixIndent(String content) { - final String[] lines = content.split("\n"); - int count = 0; - for (int i = 0; i < lines.length; i++) { - if (Pattern.compile("(\\{|\\[)$").matcher(lines[i]).find()) { - lines[i] = indent(lines[i], count); - ++count; - } else if (Pattern.compile("(\\}|\\])$").matcher(lines[i]).find()) { - --count; - lines[i] = indent(lines[i], count); - // Only indent line if previous line ended on relevant control char. - } else if (i > 0 && Pattern.compile("(=>\\s+\".+\"|,|\\{|\\}|\\[|\\])$").matcher(lines[i - 1]).find()) { - lines[i] = indent(lines[i], count); - } - } - - return String.join("\n", lines); - } - - private static String indent(String content, int shifts) { - StringBuilder spacing = new StringBuilder(); - for (int i = 0; i < shifts * 3; i++) { - spacing.append(" "); - } - return spacing.append(content).toString(); - } - - /** - * Converts Ingest/JSON style pattern array to LS pattern array, performing necessary variable - * name and quote escaping adjustments. - * @param patterns Pattern Array in JSON formatting - * @return Pattern array in LS formatting - */ - public static String createPatternArray(String... patterns) { - final String body = Arrays.stream(patterns) - .map(IngestConverter::dotsToSquareBrackets) - .map(IngestConverter::quoteString) - .collect(Collectors.joining(",\n")); - return "[\n" + body + "\n]"; - } - - public static String createArray(List ingestArray) { - final String body = ingestArray.stream() - .map(IngestConverter::quoteString) - .collect(Collectors.joining(",\n")); - return "[\n" + body + "\n]"; - } - - - /** - * Converts Ingest/JSON style pattern array to LS pattern array or string if the given array - * contains a single element only, performing necessary variable name and quote escaping - * adjustments. - * @param patterns Pattern Array in JSON formatting - * @return Pattern array or string in LS formatting - */ - public static String createPatternArrayOrField(String... patterns) { - return patterns.length == 1 - ? quoteString(dotsToSquareBrackets(patterns[0])) - : createPatternArray(patterns); - } - - public static String filterHash(String contents) { - return fixIndent(createHash("filter", contents)); - } - - public static String filtersToFile(String... filters) { - return String.join("\n\n", filters) + "\n"; - } - - /** - * Does it have an on_failure field? - * @param processor Json - * @param name Name of the processor - * @return true if has on failure - */ - @SuppressWarnings("rawtypes") - public static boolean hasOnFailure(Map processor, String name) { - final List onFailure = (List) processor.get(name).get("on_failure"); - return onFailure != null && !onFailure.isEmpty(); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public static List> getOnFailure(Map processor, String name) { - return (List>) processor.get(name).get("on_failure"); - } - - /** - * Creates an if clause with the tag name - * @param tag String tag name to find in [tags] field - * @param onFailurePipeline The on failure pipeline converted to LS to tack on in the conditional - * @return a string representing a conditional logic - */ - public static String createTagConditional(String tag, String onFailurePipeline) { - return "if " + quoteString(tag) + " in [tags] {\n" + - onFailurePipeline + "\n" + - "}"; - } - - public static String getElasticsearchOutput() { - return fixIndent("output {\n" + - "elasticsearch {\n" + - "hosts => \"localhost\"\n" + - "}\n" + - "}"); - } - - public static String getStdinInput() { - return fixIndent("input {\n" + - "stdin {\n" + - "}\n" + - "}"); - } - - public static String getStdoutOutput() { - return fixIndent("output {\n" + - "stdout {\n" + - "codec => \"rubydebug\"\n" + - "}\n" + - "}"); - } - - public static String appendIoPlugins(List filtersPipeline, boolean appendStdio) { - // TODO create unique list to join all - String filtersPipelineStr = String.join("\n", filtersPipeline); - if (appendStdio) { - return String.join("\n", IngestConverter.getStdinInput(), filtersPipelineStr, IngestConverter.getStdoutOutput()); - } else { - return String.join("\n", filtersPipelineStr, IngestConverter.getElasticsearchOutput()); - } - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java deleted file mode 100644 index 6aea685e815..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestDate { - - /** - * Converts Ingest Date JSON to LS Date filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestDate::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("date", dateHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String dateHash(Map processor) { - Map date_json = processor.get("date"); - List formats = (List) date_json.get("formats"); - - final String firstElem = IngestConverter.dotsToSquareBrackets((String) date_json.get("field")); - List match_contents = new ArrayList<>(); - match_contents.add(firstElem); - for (String f : formats) { - match_contents.add(f); - } - String date_contents = IngestConverter.createField( - "match", - IngestConverter.createPatternArray(match_contents.toArray(new String[0]))); - if (JsUtil.isNotEmpty((String) date_json.get("target_field"))) { - String target = IngestConverter.createField( - "target", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) date_json.get("target_field")) - ) - ); - date_contents = IngestConverter.joinHashFields(date_contents, target); - } - if (JsUtil.isNotEmpty((String) date_json.get("timezone"))) { - String timezone = IngestConverter.createField( - "timezone", - IngestConverter.quoteString((String) date_json.get("timezone")) - ); - date_contents = IngestConverter.joinHashFields(date_contents, timezone); - } - if (JsUtil.isNotEmpty((String) date_json.get("locale"))) { - String locale = IngestConverter.createField( - "locale", - IngestConverter.quoteString((String) date_json.get("locale")) - ); - date_contents = IngestConverter.joinHashFields(date_contents, locale); - } - return date_contents; - } - - public static boolean has_date(Map processor) { - return processor.containsKey("date"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java deleted file mode 100644 index 3394ba52d39..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestGeoIp { - - - /** - * Converts Ingest Date JSON to LS Date filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestGeoIp::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("geoip", geoIpHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String geoIpHash(Map processor) { - Map geoip_data = processor.get("geoip"); - final String sourceField = IngestConverter.createField( - "source", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) geoip_data.get("field")) - ) - ); - - final String targetField = IngestConverter.createField( - "target", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) geoip_data.get("target_field")) - ) - ); - - if (geoip_data.containsKey("properties")) { - String fields = IngestConverter.createField( - "fields", - IngestConverter.createPatternArray(((List) geoip_data.get("properties")).toArray(new String[0]) - )); - return IngestConverter.joinHashFields(sourceField, targetField, fields); - } else { - return IngestConverter.joinHashFields(sourceField, targetField); - } - } - - public static boolean has_geoip(Map processor) { - return processor.containsKey("geoip"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java deleted file mode 100644 index 9c8c8067b69..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestGrok { - - /** - * Converts Ingest JSON to LS Grok. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestGrok::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("grok", grokHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String grokHash(Map processor) { - Map grok_data = processor.get("grok"); - String grok_contents = createHashField("match", - IngestConverter.createField( - IngestConverter.quoteString((String) grok_data.get("field")), - IngestConverter.createPatternArrayOrField(((List) grok_data.get("patterns")).toArray(new String[0])) - )); - if (grok_data.containsKey("pattern_definitions")) { - grok_contents = IngestConverter.joinHashFields( - grok_contents, - createPatternDefinitionHash((Map) grok_data.get("pattern_definitions")) - ); - } - return grok_contents; - } - - private static String createHashField(String name, String content) { - return IngestConverter.createField(name, IngestConverter.wrapInCurly(content)); - } - - private static String createPatternDefinitionHash(Map definitions) { - List content = new ArrayList<>(); - for(Map.Entry entry : definitions.entrySet()) { - content.add(IngestConverter.createField( - IngestConverter.quoteString(entry.getKey()), - IngestConverter.quoteString(entry.getValue()))); - } - - final String patternDefs = content.stream().map(IngestConverter::dotsToSquareBrackets) - .collect(Collectors.joining("\n")); - - return createHashField( - "pattern_definitions", - patternDefs - ); - } - - public static boolean has_grok(Map processor) { - return processor.containsKey(get_name()); - } - - public static String get_name() { - return "grok"; - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java deleted file mode 100644 index 1ff3e3b51f3..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestGsub { - - /** - * Converts Ingest JSON to LS Grok. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestGsub::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", gsubHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String gsubHash(Map processor) { - Map gsub_data = processor.get("gsub"); - final String body = String.join(", ", - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) gsub_data.get("field"))), - IngestConverter.quoteString((String) gsub_data.get("pattern")), - IngestConverter.quoteString((String) gsub_data.get("replacement"))); - - return IngestConverter.createField("gsub", "[\n" + body + "\n]"); - } - - public static boolean has_gsub(Map processor) { - return processor.containsKey("gsub"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java deleted file mode 100644 index 77a9b37df62..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestJson { - - /** - * Converts Ingest json processor to LS json filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestJson::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("json", jsonHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String jsonHash(Map processor) { - Map json_data = processor.get("json"); - - List parts = new ArrayList(); - parts.add(IngestConverter.createField("source", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) json_data.get("field")) - ) - )); - - if (json_data.containsKey("target_field")) { - parts.add(IngestConverter.createField( - "target", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) json_data.get("target_field")) - ) - )); - } - return IngestConverter.joinHashFields(parts.toArray(new String[0])); - } - - public static boolean has_json(Map processor) { - return processor.containsKey("json"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java deleted file mode 100644 index 09b3c703a3c..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestLowercase { - - /** - * Converts Ingest Lowercase JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestLowercase::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", lowercaseHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String lowercaseHash(Map processor) { - Map lowercase_data = processor.get("lowercase"); - return IngestConverter.createField( - "lowercase", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) lowercase_data.get("field")) - ) - ); - } - - public static boolean has_lowercase(Map processor) { - return processor.containsKey("lowercase"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java deleted file mode 100644 index 1084f680c8c..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestPipeline { - - /** - * Converts Ingest JSON to LS. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestPipeline::mapProcessor).collect(Collectors.toList()); - - String logstash_pipeline = IngestConverter.filterHash( - IngestConverter.joinHashFields(filters_pipeline.toArray(new String[0]))); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(Collections.singletonList(logstash_pipeline), appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - List filter_blocks = new ArrayList<>(); - if (IngestGrok.has_grok(processor)) { - filter_blocks.add(IngestConverter.createHash(IngestGrok.get_name(), IngestGrok.grokHash(processor))); - - if (IngestConverter.hasOnFailure(processor, IngestGrok.get_name())) { - filter_blocks.add( - handle_on_failure_pipeline( - IngestConverter.getOnFailure(processor, IngestGrok.get_name()), - "_grokparsefailure" - ) - ); - } - } - boolean processed = false; - if (IngestDate.has_date(processor)) { - filter_blocks.add( - IngestConverter.createHash("date", IngestDate.dateHash(processor)) - ); - processed = true; - } - if (IngestGeoIp.has_geoip(processor)) { - filter_blocks.add( - IngestConverter.createHash("geoip", IngestGeoIp.geoIpHash(processor)) - ); - processed = true; - } - if (IngestConvert.has_convert(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestConvert.convertHash(processor)) - ); - processed = true; - } - if (IngestGsub.has_gsub(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestGsub.gsubHash(processor)) - ); - processed = true; - } - if (IngestAppend.has_append(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestAppend.appendHash(processor)) - ); - processed = true; - } - if (IngestJson.has_json(processor)) { - filter_blocks.add( - IngestConverter.createHash("json", IngestJson.jsonHash(processor)) - ); - processed = true; - } - if (IngestRename.has_rename(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestRename.renameHash(processor)) - ); - processed = true; - } - if (IngestLowercase.has_lowercase(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestLowercase.lowercaseHash(processor)) - ); - processed = true; - } - if (IngestSet.has_set(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestSet.setHash(processor)) - ); - processed = true; - } - if (!processed) { - System.out.println("WARN Found unrecognized processor named: " + processor.keySet().iterator().next()); - } - return IngestConverter.joinHashFields(filter_blocks.toArray(new String[0])); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String handle_on_failure_pipeline(List on_failure_json, String tag_name) { - final List mapped = on_failure_json.stream().map(IngestPipeline::mapProcessor).collect(Collectors.toList()); - return IngestConverter.createTagConditional(tag_name, - IngestConverter.joinHashFields(mapped.toArray(new String[0])) - ); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java deleted file mode 100644 index a21c81f53a8..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestRename { - - /** - * Converts Ingest Rename JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestRename::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", renameHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String renameHash(Map processor) { - Map rename_json = processor.get("rename"); - final String mutateContents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) rename_json.get("field"))), - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) rename_json.get("target_field"))) - ); - return IngestConverter.createField("rename", IngestConverter.wrapInCurly(mutateContents)); - } - - public static boolean has_rename(Map processor) { - return processor.containsKey("rename"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java deleted file mode 100644 index 044d787ed87..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestSet { - - /** - * Converts Ingest Set JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestSet::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", setHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String setHash(Map processor) { - Map set_json = processor.get("set"); - final Object value = set_json.get("value"); - final Object value_contents; - if (value instanceof String) { - value_contents = IngestConverter.quoteString((String) value); - } else { - value_contents = value; - } - if (set_json.containsKey("if") && set_json.get("if") != null) { - String painless_condition = (String) set_json.get("if"); - if (!painless_condition.isEmpty()) { - System.out.println("WARN Found in 'set' processor an 'if' painless condition not translated: " + painless_condition); - } - - } - - String mutate_contents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) set_json.get("field"))), - value_contents.toString()); - return IngestConverter.createField("add_field", IngestConverter.wrapInCurly(mutate_contents)); - } - - public static boolean has_set(Map processor) { - return processor.containsKey("set"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java deleted file mode 100644 index 806c2f36f27..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import javax.script.Invocable; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; -import joptsimple.OptionException; -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import joptsimple.OptionSpec; - -final class JsUtil { - - /** - * Script names used by the converter in correct load order. - */ - - private static final String[] SCRIPTS = { - "shared", "date", "grok", "geoip", "gsub", "pipeline", "convert", "append", "json", - "rename", "lowercase", "set" - }; - - private JsUtil() { - // Utility Class - } - - /** - * Sets up a {@link ScriptEngine} with all Ingest to LS DSL Converter JS scripts loaded. - * @return {@link ScriptEngine} for Ingest to LS DSL Converter - */ - public static ScriptEngine engine() { - final ScriptEngine engine = - new ScriptEngineManager().getEngineByName("nashorn"); - try { - for (final String file : SCRIPTS) { - add(engine, String.format("/ingest-%s.js", file)); - } - } catch (final IOException | ScriptException ex) { - throw new IllegalStateException(ex); - } - return engine; - } - - /** - * Converts the given files from ingest to LS conf using the javascript function - * @param args CLI Arguments - * @param jsFunc JS function to call - * @throws ScriptException - * @throws NoSuchMethodException - */ - public static void convert(final String[] args, final String jsFunc) - throws ScriptException, NoSuchMethodException { - final OptionParser parser = new OptionParser(); - final OptionSpec input = parser.accepts( - "input", - "Input JSON file location URI. Only supports 'file://' as URI schema." - ).withRequiredArg().ofType(URI.class).required().forHelp(); - final OptionSpec output = parser.accepts( - "output", - "Output Logstash DSL file location URI. Only supports 'file://' as URI schema." - ).withRequiredArg().ofType(URI.class).required().forHelp(); - final OptionSpec appendStdio = parser.accepts( - "append-stdio", - "Flag to append stdin and stdout as outputs instead of the default ES output." - ).forHelp(); - try { - final OptionSet options; - try { - options = parser.parse(args); - } catch (final OptionException ex) { - parser.printHelpOn(System.out); - throw ex; - } - switch (jsFunc) { - case "ingest_append_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestAppend.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_convert_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestConvert.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_date": - Files.write( - Paths.get(options.valueOf(output)), - IngestDate.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_geoip": - Files.write( - Paths.get(options.valueOf(output)), - IngestGeoIp.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_grok": - Files.write( - Paths.get(options.valueOf(output)), - IngestGrok.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_gsub": - Files.write( - Paths.get(options.valueOf(output)), - IngestGsub.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_json_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestJson.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_lowercase_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestLowercase.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_rename_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestRename.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_set_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestSet.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_pipeline_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestPipeline.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - - default: { - throw new IllegalArgumentException("Can't recognize " + jsFunc + " processor"); - } - } - - } catch (final IOException ex) { - throw new IllegalStateException(ex); - } - } - - /** - * Retrieves the input Ingest JSON from a given {@link URI}. - * @param uri {@link URI} of Ingest JSON - * @return Json String - * @throws IOException On failure to load Ingest JSON - */ - private static String input(final URI uri) throws IOException { - if ("file".equals(uri.getScheme())) { - return new String( - Files.readAllBytes(Paths.get(uri)), StandardCharsets.UTF_8 - ); - } - throw new IllegalArgumentException("--input must be of schema file://"); - } - - private static void add(final ScriptEngine engine, final String file) - throws IOException, ScriptException { - try (final Reader reader = - new InputStreamReader(JsUtil.class.getResourceAsStream(file))) { - engine.eval(reader); - } - } - - /*** - * Not empty check with nullability - * @param s string to check - * @return true iff s in not null and not empty - */ - static boolean isNotEmpty(String s) { - return s != null && !s.isEmpty(); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java deleted file mode 100644 index 21b10d856ce..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest JSON processor DSL to Logstash json Transpiler. - */ -public class Json { - private Json() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_json_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java deleted file mode 100644 index 77ccb8c64ec..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Lowercase DSL to Logstash mutate Transpiler. - */ -public final class Lowercase { - - private Lowercase() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_lowercase_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java deleted file mode 100644 index e888852f749..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Full DSL to Logstash DSL Transpiler. - */ -public final class Pipeline { - - private Pipeline() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_pipeline_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java deleted file mode 100644 index dac5129a0ce..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -public class Rename { - private Rename() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_rename_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java deleted file mode 100644 index 5872958692d..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Set DSL to Logstash mutate Transpiler. - */ -public class Set { - private Set() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_set_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java deleted file mode 100644 index 0fa5cd43d9d..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class AppendTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Append", "DotsInAppendField", "AppendScalar"); - } - - @Test - public void convertsAppendProcessorCorrectly() throws Exception { - assertCorrectConversion(Append.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java deleted file mode 100644 index e2197e116c6..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class ConvertTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Convert", "DotsInConvertField", "ConvertBoolean", "ConvertString"); - } - - @Test - public void convertsConvertProcessorCorrectly() throws Exception { - assertCorrectConversion(Convert.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java deleted file mode 100644 index 49a475d2a8d..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class DateTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Date", "DateExtraFields", "DotsInDateField"); - } - - @Test - public void convertsDateFieldCorrectly() throws Exception { - assertCorrectConversion(Date.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java deleted file mode 100644 index fb83ad3be1e..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class GeoIpTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("GeoIpSimple", "DotsInGeoIpField"); - } - - @Test - public void convertsGeoIpFieldCorrectly() throws Exception { - assertCorrectConversion(GeoIp.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java deleted file mode 100644 index d423baf5f80..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class GrokTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Grok", "GrokPatternDefinition", "GrokMultiplePatternDefinitions"); - } - - @Test - public void convertsGrokFieldCorrectly() throws Exception { - assertCorrectConversion(Grok.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java deleted file mode 100644 index 26f422c136a..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Collections; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class GsubTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Collections.singletonList("GsubSimple"); - } - - @Test - public void convertsGsubCorrectly() throws Exception { - assertCorrectConversion(Gsub.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java deleted file mode 100644 index d8983abe14a..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.regex.Pattern; -import org.apache.commons.io.IOUtils; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.runners.Parameterized.Parameter; - -/** - * Base class for ingest migration tests - */ -@RunWith(Parameterized.class) -public abstract class IngestTest { - - /** - * Used to normalize line endings since static reference result files have Unix line endings. - */ - private static final Pattern CR_LF = - Pattern.compile("\\r\\n"); - - /** - * Used to normalize line endings since static reference result files have Unix line endings. - */ - private static final Pattern CARRIAGE_RETURN = Pattern.compile("\\r"); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Parameter - public String testCase; - - protected final void assertCorrectConversion(final Class clazz) throws Exception { - final URL append = getResultPath(temp); - clazz.getMethod("main", String[].class).invoke( - null, - (Object) new String[]{ - String.format("--input=%s", resourcePath(String.format("ingest%s.json", testCase))), - String.format("--output=%s", append) - } - ); - assertThat( - utf8File(append), is(utf8File(resourcePath(String.format("logstash%s.conf", testCase)))) - ); - } - - /** - * Reads a file, normalizes line endings to Unix line endings and returns the whole content - * as a String. - * @param path Url to read - * @return String content of the URL - * @throws IOException On failure to read from given URL - */ - private static String utf8File(final URL path) throws IOException { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (final InputStream input = path.openStream()) { - IOUtils.copy(input, baos); - } - return CARRIAGE_RETURN.matcher( - CR_LF.matcher( - baos.toString(StandardCharsets.UTF_8.name()) - ).replaceAll("\n") - ).replaceAll("\n"); - } - - private static URL resourcePath(final String name) { - return IngestTest.class.getResource(name); - } - - private static URL getResultPath(TemporaryFolder temp) throws IOException { - return temp.newFolder().toPath().resolve("converted").toUri().toURL(); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java deleted file mode 100644 index cb7bf65b087..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class JsonTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Json", "DotsInJsonField", "JsonExtraFields"); - } - - @Test - public void convertsConvertProcessorCorrectly() throws Exception { - assertCorrectConversion(Json.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java deleted file mode 100644 index e6231dc6080..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class LowercaseTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("LowercaseSimple", "LowercaseDots"); - } - - @Test - public void convertsAppendProcessorCorrectly() throws Exception { - assertCorrectConversion(Lowercase.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java deleted file mode 100644 index 830f5c466a0..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.ArrayList; -import java.util.Collection; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class PipelineTest extends IngestTest { - - @Parameters - public static Iterable data() { - final Collection cases = new ArrayList<>(); - cases.add("ComplexCase1"); - cases.add("ComplexCase2"); - cases.add("ComplexCase3"); - cases.add("ComplexCase4"); - GeoIpTest.data().forEach(cases::add); - DateTest.data().forEach(cases::add); - GrokTest.data().forEach(cases::add); - ConvertTest.data().forEach(cases::add); - GsubTest.data().forEach(cases::add); - AppendTest.data().forEach(cases::add); - JsonTest.data().forEach(cases::add); - RenameTest.data().forEach(cases::add); - LowercaseTest.data().forEach(cases::add); - SetTest.data().forEach(cases::add); - return cases; - } - - @Test - public void convertsComplexCaseCorrectly() throws Exception { - assertCorrectConversion(Pipeline.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java deleted file mode 100644 index 2ecd074eb2e..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class RenameTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Rename", "DotsInRenameField"); - } - - @Test - public void convertsConvertProcessorCorrectly() throws Exception { - assertCorrectConversion(Rename.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java deleted file mode 100644 index 8682b616cd6..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class SetTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Set", "DotsInSetField", "SetNumber"); - } - - @Test - public void convertsSetProcessorCorrectly() throws Exception { - assertCorrectConversion(Set.class); - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json deleted file mode 100644 index 471d4337cd2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "append": { - "field" : "client", - "value": ["host1", "host2"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json deleted file mode 100644 index 34ad59e5e7d..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "append": { - "field" : "foo", - "value": "bar" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json deleted file mode 100644 index 370fd72d604..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{COMBINEDAPACHELOG}" - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "client.ip", - "target_field": "geo" - } - }, - { - "convert": { - "field" : "bytes", - "type": "integer" - } - }, - { - "append": { - "field" : "response_code", - "value": ["200", "400", "503"] - } - }, - { - "json": { - "field": "string_source" - } - }, - { - "rename": { - "field": "foo", - "target_field": "foobar" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json deleted file mode 100644 index 054f6e3fbb2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{COMBINEDAPACHELOG}" - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "client.ip", - "target_field": "client.geo" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo" - } - }, - { - "convert": { - "field" : "[client][bytes]", - "type": "integer" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json deleted file mode 100644 index d059b3536de..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": ["%{COMBINEDAPACHELOG}"], - "on_failure" : [ - { - "set" : { - "field" : "error", - "value" : "field does not exist" - } - } - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "clientip", - "target_field": "geo" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json deleted file mode 100644 index c63bda67333..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": ["%{COMBINEDAPACHELOG}"], - "on_failure" : [ - { - "set" : { - "field" : "error", - "value" : "field does not exist" - } - }, - { - "convert": { - "field" : "client.ip", - "type": "integer" - } - } - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "clientip", - "target_field": "geo" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json deleted file mode 100644 index 11b730dcdad..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "processors": [ - { - "convert": { - "field" : "bytes", - "type": "integer" - } - } -] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json deleted file mode 100644 index 3904f5ab88d..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "processors": [ - { - "convert": { - "field" : "delete", - "type": "boolean" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json deleted file mode 100644 index e61614c5648..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "processors": [ - { - "convert": { - "field" : "blah", - "type": "string" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json deleted file mode 100644 index b93104bc28b..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description" : "...", - "processors" : [ - { - "date" : { - "field" : "initial_date", - "target_field" : "timestamp", - "formats" : ["dd/MM/yyyy hh:mm:ss", "dd/MM/yyyy"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json deleted file mode 100644 index 8a342f95a47..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "description" : "...", - "processors" : [ - { - "date" : { - "field" : "initial_date", - "target_field" : "timestamp", - "formats" : ["dd/MM/yyyy hh:mm:ss", "dd/MM/yyyy"], - "timezone" : "Europe/Amsterdam", - "locale": "en" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json deleted file mode 100644 index 70989b3546d..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "append": { - "field" : "client.ip", - "value": ["127.0.0.1", "127.0.0.2"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json deleted file mode 100644 index c8ac73b1b0a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "convert": { - "field" : "client.bytes", - "type": "float" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json deleted file mode 100644 index 4a9a3576604..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "description" : "...", - "processors" : [ - { - "date" : { - "field" : "initial_date", - "target_field" : "apache.timestamp", - "formats" : ["dd/MM/yyyy hh:mm:ss", "dd/MM/yyyy"], - "timezone" : "Europe/Amsterdam", - "locale": "en" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json deleted file mode 100644 index 4a3b7fe18d3..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "description" : "Add geoip info", - "processors" : [ - { - "geoip" : { - "field" : "ip", - "target_field" : "apache.geo", - "database_file" : "GeoLite2-Country.mmdb.gz", - "properties": ["continent_name", "country_iso_code"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json deleted file mode 100644 index ed4f41f1f34..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleJson", - "processors": [ - { - "json": { - "field": "[foo][string_source]", - "target_field": "[bar][json_target]" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json deleted file mode 100644 index 33aa582a655..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleRename", - "processors": [ - { - "rename": { - "field": "foo.bar", - "target_field": "foo.baz" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json deleted file mode 100644 index 7112b7131b5..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "SetExample", - "processors": [ - { - "set": { - "field": "foo.bar", - "value": "baz" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json deleted file mode 100644 index 2554b92c474..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "description": "Add geoip info", - "processors": [ - { - "geoip": { - "field": "ip", - "target_field": "geo", - "database_file": "GeoLite2-Country.mmdb.gz", - "properties": [ - "continent_name", - "country_iso_code" - ] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json deleted file mode 100644 index 36db975a177..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "description": "Pipeline for parsing apache error logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"%{WORD:apache2.access.method} %{DATA:apache2.access.url} HTTP/%{NUMBER:apache2.access.http_version}\" %{NUMBER:apache2.access.response_code} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( \"%{DATA:apache2.access.referrer}\")?( \"%{DATA:apache2.access.agent}\")?", - "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"-\" %{NUMBER:apache2.access.response_code} -" - ], - "ignore_missing": true - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json deleted file mode 100644 index e3a2ce60eda..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "description":"Syslog", - "processors":[ - { - "grok":{ - "field":"message", - "patterns":[ - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}", - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" - ], - "pattern_definitions":{ - "GREEDYMULTILINE":"(.|\\n)*", - "AUDIT_TYPE": "^type=%{NOTSPACE:auditd.log.record_type}" - }, - "ignore_missing":true - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json deleted file mode 100644 index bb123eed1b5..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "description":"Syslog", - "processors":[ - { - "grok":{ - "field":"message", - "patterns":[ - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}", - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" - ], - "pattern_definitions":{ - "GREEDYMULTILINE":"(.|\\n)*" - }, - "ignore_missing":true - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json deleted file mode 100644 index 14835baa028..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description": "ExampleGsub", - "processors": [ - { - "gsub": { - "field": "field1", - "pattern": "\\.", - "replacement": "_" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json deleted file mode 100644 index 54a8adf6609..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "description": "ExampleJson", - "processors": [ - { - "json": { - "field": "string_source" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json deleted file mode 100644 index e95bedc274a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleJson", - "processors": [ - { - "json": { - "field": "string_source", - "target_field": "json_target" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json deleted file mode 100644 index 2f375b36ef3..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "description": "ExampleLowercase", - "processors": [ - { - "lowercase": { - "field": "foo.bar" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json deleted file mode 100644 index 8ae738f1961..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "description": "ExampleLowercase", - "processors": [ - { - "lowercase": { - "field": "foo" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json deleted file mode 100644 index 7a2d89ed4ed..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleRename", - "processors": [ - { - "rename": { - "field": "foo", - "target_field": "foobar" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json deleted file mode 100644 index 3ae6f3c3912..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description": "SetExample", - "processors": [ - { - "set": { - "field": "field1", - "value": "bar" - } - } - ] -} - diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json deleted file mode 100644 index 70558bd5255..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "SetExample", - "processors": [ - { - "set": { - "field": "field1", - "value": 5344.4 - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf deleted file mode 100644 index 40372d15cc4..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - mutate { - add_field => { - "client" => [ - "host1", - "host2" - ] - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf deleted file mode 100644 index f40d36e3810..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "foo" => "bar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf deleted file mode 100644 index be27a7bc6f2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf +++ /dev/null @@ -1,46 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "[client][ip]" - target => "geo" - } - mutate { - convert => { - "bytes" => "integer" - } - } - mutate { - add_field => { - "response_code" => [ - "200", - "400", - "503" - ] - } - } - json { - source => "string_source" - } - mutate { - rename => { - "foo" => "foobar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf deleted file mode 100644 index 20b94771d0a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf +++ /dev/null @@ -1,33 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "[client][ip]" - target => "[client][geo]" - } - geoip { - source => "[source][ip]" - target => "[source][geo]" - } - mutate { - convert => { - "[client][bytes]" => "integer" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf deleted file mode 100644 index 20ae4380e4b..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf +++ /dev/null @@ -1,31 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - if "_grokparsefailure" in [tags] { - mutate { - add_field => { - "error" => "field does not exist" - } - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "clientip" - target => "geo" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf deleted file mode 100644 index 4da212f20bd..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf +++ /dev/null @@ -1,36 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - if "_grokparsefailure" in [tags] { - mutate { - add_field => { - "error" => "field does not exist" - } - } - mutate { - convert => { - "[client][ip]" => "integer" - } - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "clientip" - target => "geo" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf deleted file mode 100644 index 1771fe204c8..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "bytes" => "integer" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf deleted file mode 100644 index 0d98c5a4720..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "delete" => "boolean" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf deleted file mode 100644 index 6a872cf7a7e..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "blah" => "string" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf deleted file mode 100644 index 0533a7f5296..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - date { - match => [ - "initial_date", - "dd/MM/yyyy hh:mm:ss", - "dd/MM/yyyy" - ] - target => "timestamp" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf deleted file mode 100644 index 52cb3e471a3..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf +++ /dev/null @@ -1,17 +0,0 @@ -filter { - date { - match => [ - "initial_date", - "dd/MM/yyyy hh:mm:ss", - "dd/MM/yyyy" - ] - target => "timestamp" - timezone => "Europe/Amsterdam" - locale => "en" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf deleted file mode 100644 index 8ee17721312..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - mutate { - add_field => { - "[client][ip]" => [ - "127.0.0.1", - "127.0.0.2" - ] - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf deleted file mode 100644 index 8bbe66ce5b7..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "[client][bytes]" => "float" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf deleted file mode 100644 index 881d9e47b06..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf +++ /dev/null @@ -1,17 +0,0 @@ -filter { - date { - match => [ - "initial_date", - "dd/MM/yyyy hh:mm:ss", - "dd/MM/yyyy" - ] - target => "[apache][timestamp]" - timezone => "Europe/Amsterdam" - locale => "en" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf deleted file mode 100644 index fbcd2b2026a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - geoip { - source => "ip" - target => "[apache][geo]" - fields => [ - "continent_name", - "country_iso_code" - ] - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf deleted file mode 100644 index 57032dfcb95..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf +++ /dev/null @@ -1,11 +0,0 @@ -filter { - json { - source => "[foo][string_source]" - target => "[bar][json_target]" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf deleted file mode 100644 index aedde2200af..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - rename => { - "[foo][bar]" => "[foo][baz]" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf deleted file mode 100644 index 3b63d8f0fa2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "[foo][bar]" => "baz" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf deleted file mode 100644 index 218e672c5b5..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - geoip { - source => "ip" - target => "geo" - fields => [ - "continent_name", - "country_iso_code" - ] - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf deleted file mode 100644 index ddf6150c10a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - grok { - match => { - "message" => [ - "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?", - "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"-\" %{NUMBER:[apache2][access][response_code]} -" - ] - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf deleted file mode 100644 index e89065b7138..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf +++ /dev/null @@ -1,19 +0,0 @@ -filter { - grok { - match => { - "message" => [ - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}", - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{GREEDYMULTILINE:[system][syslog][message]}" - ] - } - pattern_definitions => { - "GREEDYMULTILINE" => "(.|\n)*" - "AUDIT_TYPE" => "^type=%{NOTSPACE:[auditd][log][record_type]}" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf deleted file mode 100644 index 0a24c2676bb..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf +++ /dev/null @@ -1,18 +0,0 @@ -filter { - grok { - match => { - "message" => [ - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}", - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{GREEDYMULTILINE:[system][syslog][message]}" - ] - } - pattern_definitions => { - "GREEDYMULTILINE" => "(.|\n)*" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf deleted file mode 100644 index bc4092a8070..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - gsub => [ - "field1", "\.", "_" - ] - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf deleted file mode 100644 index 1936d3c3453..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - json { - source => "string_source" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf deleted file mode 100644 index dfe7c2d48d0..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf +++ /dev/null @@ -1,11 +0,0 @@ -filter { - json { - source => "string_source" - target => "json_target" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf deleted file mode 100644 index 9f070e3678a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - mutate { - lowercase => "[foo][bar]" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf deleted file mode 100644 index 49229ab26a6..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - mutate { - lowercase => "foo" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf deleted file mode 100644 index daf5725ec25..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - rename => { - "foo" => "foobar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf deleted file mode 100644 index 204cb54240f..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "field1" => "bar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf deleted file mode 100644 index 36a98a3f567..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "field1" => 5344.4 - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java index 0e4d0cf5152..7aa3f484d23 100644 --- a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java +++ b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java @@ -30,10 +30,10 @@ public class JavaVersion implements Comparable { public static final JavaVersion CURRENT = parse(System.getProperty("java.specification.version")); - public static final JavaVersion JAVA_11 = parse("11"); + public static final JavaVersion JAVA_17 = parse("17"); private final List version; - private JavaVersion(List version){ + private JavaVersion(List version) { this.version = version; } diff --git a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java index 84799da1ca5..d19b5f547e7 100644 --- a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java +++ b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java @@ -114,11 +114,11 @@ public static void main(final String[] args) { handleJvmOptions(args, System.getenv("LS_JAVA_OPTS")); } - static void bailOnOldJava(){ - if (JavaVersion.CURRENT.compareTo(JavaVersion.JAVA_11) < 0) { + static void bailOnOldJava() { + if (JavaVersion.CURRENT.compareTo(JavaVersion.JAVA_17) < 0) { final String message = String.format( Locale.ROOT, - "The minimum required Java version is 11; your Java version from [%s] does not meet this requirement", + "The minimum required Java version is 17; your Java version from [%s] does not meet this requirement", System.getProperty("java.home") ); System.err.println(message); diff --git a/versions.yml b/versions.yml index 4626e47ca9b..f8249cb1965 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 8.16.0 -logstash-core: 8.16.0 +logstash: 9.0.0 +logstash-core: 9.0.0 logstash-core-plugin-api: 2.1.16 bundled_jdk: diff --git a/x-pack/qa/integration/support/helpers.rb b/x-pack/qa/integration/support/helpers.rb index 06069f3a1d3..a216d1cd8d4 100644 --- a/x-pack/qa/integration/support/helpers.rb +++ b/x-pack/qa/integration/support/helpers.rb @@ -48,7 +48,7 @@ def elasticsearch(options = {}) # Launch in the background and wait for /started/ stdout cmd = "bin/elasticsearch #{settings_arguments.join(' ')}" puts "Running elasticsearch: #{cmd}" - response = Belzebuth.run(cmd, { :directory => get_elasticsearch_path, :wait_condition => /license.*valid/, :timeout => 15 * 60 }) + response = Belzebuth.run(cmd, { :directory => get_elasticsearch_path, :wait_condition => /ClusterStateLicenseService.*license.*valid/, :timeout => 15 * 60 }) unless response.successful? raise "Could not start Elasticsearch, response: #{response}" end