From c4c8b7f0ea9d17eb3ecc41b00e42edd120456671 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 10:44:54 +0100 Subject: [PATCH 1/9] Test: reproduce double :port info from ES client --- spec/inputs/integration/elasticsearch_spec.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index e7c1132..d107ec8 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -6,7 +6,9 @@ describe LogStash::Inputs::Elasticsearch do - let(:config) { { 'hosts' => [ESHelper.get_host_port], + SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true' + + let(:config) { { 'hosts' => ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"], 'index' => 'logs', 'query' => '{ "query": { "match": { "message": "Not found"} }}' } } let(:plugin) { described_class.new(config) } @@ -78,4 +80,15 @@ end end + + context 'setting host:port (and ssl)' do + + let(:config) do + super().merge "hosts" => [ESHelper.get_host_port], "ssl" => SECURE_INTEGRATION + end + + it_behaves_like 'an elasticsearch index plugin' + + end + end From e5da1c237f55541fe52e3a899306bbe5e4e5ffab Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 10:54:10 +0100 Subject: [PATCH 2/9] tag integration specs properly --- spec/inputs/integration/elasticsearch_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index d107ec8..2d9a685 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -4,7 +4,7 @@ require "logstash/inputs/elasticsearch" require_relative "../../../spec/es_helper" -describe LogStash::Inputs::Elasticsearch do +describe LogStash::Inputs::Elasticsearch, :integration => true do SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true' @@ -47,7 +47,7 @@ end end - describe 'against an unsecured elasticsearch', :integration => true do + describe 'against an unsecured elasticsearch' do before(:each) do plugin.register end From ea8bef09a6affe247de21fa5633a4ba7fc60dcbd Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 10:55:31 +0100 Subject: [PATCH 3/9] CI: more targets (against Manticore/ES versions) --- .travis.yml | 10 ++++++---- Gemfile | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2283ac..998e838 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,12 @@ before_install: env: - INTEGRATION=false ELASTIC_STACK_VERSION=7.x +- INTEGRATION=false ELASTIC_STACK_VERSION=8.x - INTEGRATION=false ELASTIC_STACK_VERSION=7.x SNAPSHOT=true - INTEGRATION=false ELASTIC_STACK_VERSION=8.x SNAPSHOT=true +- INTEGRATION=false ELASTIC_STACK_VERSION=7.16.3 MANTICORE_VERSION=0.7.1 ELASTICSEARCH_VERSION=7.15.0 - INTEGRATION=true ELASTIC_STACK_VERSION=7.x -- INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true -- INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true -- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x -- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.15.0 +- INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info +- INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info +- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info +- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.15.0 MANTICORE_VERSION=0.7.1 ELASTICSEARCH_VERSION=7.14.1 LOG_LEVEL=info diff --git a/Gemfile b/Gemfile index 32cc6fb..85bf037 100644 --- a/Gemfile +++ b/Gemfile @@ -9,3 +9,6 @@ if Dir.exist?(logstash_path) && use_logstash_source gem 'logstash-core', :path => "#{logstash_path}/logstash-core" gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" end + +gem 'manticore', ENV['MANTICORE_VERSION'] if ENV['MANTICORE_VERSION'] +gem 'elasticsearch', ENV['ELASTICSEARCH_VERSION'] if ENV['ELASTICSEARCH_VERSION'] From f7e181a9f3060616649968fcd256a976a962bd0a Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 11:18:00 +0100 Subject: [PATCH 4/9] re-tag integration specs --- .ci/logstash-run.sh | 8 ++++---- spec/inputs/integration/elasticsearch_spec.rb | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index d0fb854..b39ed0e 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -21,13 +21,13 @@ wait_for_es() { } if [[ "$INTEGRATION" != "true" ]]; then - bundle exec rspec -fd spec/inputs -t ~integration -t ~secure_integration + jruby -rbundler/setup -S rspec -fd --tag ~integration spec/inputs else if [[ "$SECURE_INTEGRATION" == "true" ]]; then - extra_tag_args="--tag secure_integration" + extra_tag_args="--tag integration:secure" else - extra_tag_args="--tag ~secure_integration --tag integration" + extra_tag_args="--tag integration" fi wait_for_es - bundle exec rspec -fd $extra_tag_args --tag es_version:$ELASTIC_STACK_VERSION spec/inputs/integration + jruby -rbundler/setup -S rspec -fd $extra_tag_args --tag es_version:$ELASTIC_STACK_VERSION spec/inputs/integration fi diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index 2d9a685..6e4624d 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -4,7 +4,7 @@ require "logstash/inputs/elasticsearch" require_relative "../../../spec/es_helper" -describe LogStash::Inputs::Elasticsearch, :integration => true do +describe LogStash::Inputs::Elasticsearch, integration: true do SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true' @@ -55,7 +55,7 @@ it_behaves_like 'an elasticsearch index plugin' end - describe 'against a secured elasticsearch', :secure_integration => true do + describe 'against a secured elasticsearch', integration: 'secure' do let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' } let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' } let(:ca_file) { "spec/fixtures/test_certs/ca.crt" } @@ -81,7 +81,7 @@ end - context 'setting host:port (and ssl)' do + context 'setting host:port (and ssl)', integration: SECURE_INTEGRATION ? 'secure' : true do let(:config) do super().merge "hosts" => [ESHelper.get_host_port], "ssl" => SECURE_INTEGRATION From 55db6b24eb3b4bcb3fad16da7a3ef8964f65d71f Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 11:41:53 +0100 Subject: [PATCH 5/9] roll like filter with one integration tag --- .ci/logstash-run.sh | 2 +- spec/inputs/integration/elasticsearch_spec.rb | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index b39ed0e..114abea 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -24,7 +24,7 @@ if [[ "$INTEGRATION" != "true" ]]; then jruby -rbundler/setup -S rspec -fd --tag ~integration spec/inputs else if [[ "$SECURE_INTEGRATION" == "true" ]]; then - extra_tag_args="--tag integration:secure" + extra_tag_args="--tag integration" # secure filtered in spec (we run scenarios both under http as well as https) else extra_tag_args="--tag integration" fi diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index 6e4624d..dc381e4 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -55,7 +55,7 @@ it_behaves_like 'an elasticsearch index plugin' end - describe 'against a secured elasticsearch', integration: 'secure' do + describe 'against a secured elasticsearch' do let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' } let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' } let(:ca_file) { "spec/fixtures/test_certs/ca.crt" } @@ -79,12 +79,18 @@ end end - end + end if SECURE_INTEGRATION + + context 'setting host:port (and ssl)' do - context 'setting host:port (and ssl)', integration: SECURE_INTEGRATION ? 'secure' : true do + let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' } + let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' } + let(:ca_file) { "spec/fixtures/test_certs/ca.crt" } let(:config) do - super().merge "hosts" => [ESHelper.get_host_port], "ssl" => SECURE_INTEGRATION + config = super().merge "hosts" => [ESHelper.get_host_port] + config.merge!('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) if SECURE_INTEGRATION + config end it_behaves_like 'an elasticsearch index plugin' From e52d6b79f9a5d1da7fa8482126a4e518d1e35360 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 11:48:08 +0100 Subject: [PATCH 6/9] restore previous secure_integration tag filter --- .ci/logstash-run.sh | 4 +-- spec/inputs/integration/elasticsearch_spec.rb | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index 114abea..bbc04b5 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -24,9 +24,9 @@ if [[ "$INTEGRATION" != "true" ]]; then jruby -rbundler/setup -S rspec -fd --tag ~integration spec/inputs else if [[ "$SECURE_INTEGRATION" == "true" ]]; then - extra_tag_args="--tag integration" # secure filtered in spec (we run scenarios both under http as well as https) + extra_tag_args="--tag secure_integration" else - extra_tag_args="--tag integration" + extra_tag_args="--tag ~secure_integration --tag integration" fi wait_for_es jruby -rbundler/setup -S rspec -fd $extra_tag_args --tag es_version:$ELASTIC_STACK_VERSION spec/inputs/integration diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index dc381e4..5ca5b84 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -15,6 +15,10 @@ let(:event) { LogStash::Event.new({}) } let(:client_options) { Hash.new } + let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' } + let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' } + let(:ca_file) { "spec/fixtures/test_certs/ca.crt" } + before(:each) do @es = ESHelper.get_client(client_options) # Delete all templates first. @@ -55,10 +59,7 @@ it_behaves_like 'an elasticsearch index plugin' end - describe 'against a secured elasticsearch' do - let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' } - let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' } - let(:ca_file) { "spec/fixtures/test_certs/ca.crt" } + describe 'against a secured elasticsearch', secure_integration: true do let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } } @@ -79,18 +80,25 @@ end end - end if SECURE_INTEGRATION + end + + context 'setting host:port' do - context 'setting host:port (and ssl)' do + let(:config) do + super().merge "hosts" => [ESHelper.get_host_port] + end - let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' } - let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' } - let(:ca_file) { "spec/fixtures/test_certs/ca.crt" } + it_behaves_like 'an elasticsearch index plugin' + + end + + context 'setting host:port (and ssl)', secure_integration: true do + + let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } } let(:config) do config = super().merge "hosts" => [ESHelper.get_host_port] - config.merge!('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) if SECURE_INTEGRATION - config + config.merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) end it_behaves_like 'an elasticsearch index plugin' From 9d006a8de13f31d71dbf82d7e0969379fe30ae71 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 12:20:21 +0100 Subject: [PATCH 7/9] tag integration/secure_integration and skip both in unit --- .ci/logstash-run.sh | 2 +- spec/inputs/integration/elasticsearch_spec.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index bbc04b5..e7e81e1 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -21,7 +21,7 @@ wait_for_es() { } if [[ "$INTEGRATION" != "true" ]]; then - jruby -rbundler/setup -S rspec -fd --tag ~integration spec/inputs + jruby -rbundler/setup -S rspec -fd --tag ~integration --tag ~secure_integration spec/inputs else if [[ "$SECURE_INTEGRATION" == "true" ]]; then extra_tag_args="--tag secure_integration" diff --git a/spec/inputs/integration/elasticsearch_spec.rb b/spec/inputs/integration/elasticsearch_spec.rb index 5ca5b84..8edc5d8 100644 --- a/spec/inputs/integration/elasticsearch_spec.rb +++ b/spec/inputs/integration/elasticsearch_spec.rb @@ -4,7 +4,7 @@ require "logstash/inputs/elasticsearch" require_relative "../../../spec/es_helper" -describe LogStash::Inputs::Elasticsearch, integration: true do +describe LogStash::Inputs::Elasticsearch do SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true' @@ -51,7 +51,7 @@ end end - describe 'against an unsecured elasticsearch' do + describe 'against an unsecured elasticsearch', integration: true do before(:each) do plugin.register end @@ -82,7 +82,7 @@ end - context 'setting host:port' do + context 'setting host:port', integration: true do let(:config) do super().merge "hosts" => [ESHelper.get_host_port] From cbe41875ba7356fba845a6f95eb91c4f86735cce Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 12:31:34 +0100 Subject: [PATCH 8/9] Fix: hosts => "es_host:port" regression --- lib/logstash/inputs/elasticsearch.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index 851ebd7..1319d7f 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -386,13 +386,13 @@ def setup_ssl def setup_hosts @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s - if @ssl - @hosts.map do |h| - host, port = h.split(":") - { :host => host, :scheme => 'https', :port => port } + @hosts.map do |h| + if h.start_with?('http:', 'https:') + h + else + host, port = h.split(':') + { host: host, port: port, scheme: (@ssl ? 'https' : 'http') } end - else - @hosts end end From d62126d5c7925d652f0aeb1824038605cc54c907 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 17 Feb 2022 12:33:36 +0100 Subject: [PATCH 9/9] bump + changelog --- CHANGELOG.md | 3 +++ logstash-input-elasticsearch.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 693ae49..2251e18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.12.2 + - Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168) + ## 4.12.1 - Fixed too_long_frame_exception by passing scroll_id in the body [#159](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/159) diff --git a/logstash-input-elasticsearch.gemspec b/logstash-input-elasticsearch.gemspec index 5570314..2cd5bfc 100644 --- a/logstash-input-elasticsearch.gemspec +++ b/logstash-input-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-elasticsearch' - s.version = '4.12.1' + s.version = '4.12.2' s.licenses = ['Apache License (2.0)'] s.summary = "Reads query results from an Elasticsearch cluster" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"