From 3b304b336756c1410637b5819fca62ba209291b1 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 29 May 2024 15:41:26 -0400 Subject: [PATCH 1/5] update elastic-agent-libs to v0.9.9 Update elastic-agent-libs to v0.9.9 and add tests to ensure Beats always logs as JSON to a file when no logging is explicitly configured. --- NOTICE.txt | 33 ++++++- filebeat/tests/integration/filebeat_test.go | 96 +++++++++++++++++++++ go.mod | 2 +- go.sum | 6 +- libbeat/tests/integration/framework.go | 12 +++ 5 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 filebeat/tests/integration/filebeat_test.go diff --git a/NOTICE.txt b/NOTICE.txt index 4e0834b9769..a9af6aa666f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.8 +Version: v0.9.9 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.9/LICENSE: Apache License Version 2.0, January 2004 @@ -55932,6 +55932,35 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L limitations under the License. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/mcuadros/go-syslog.v2 +Version: v2.3.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/mcuadros/go-syslog.v2@v2.3.0/LICENSE: + +Copyright (c) 2013 Máximo Cuadros + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : gopkg.in/tomb.v1 Version: v1.0.0-20141024135613-dd632973f1e7 diff --git a/filebeat/tests/integration/filebeat_test.go b/filebeat/tests/integration/filebeat_test.go new file mode 100644 index 00000000000..a88631bdbea --- /dev/null +++ b/filebeat/tests/integration/filebeat_test.go @@ -0,0 +1,96 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package integration + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +var filebeatBasicConfig = ` +filebeat.inputs: + - type: filestream + id: "test-filebeat-can-log" + paths: + - %s +path.home: %s +output.discard.enabled: true +` + +func TestFilebeatRunsAndLogsJSONToFile(t *testing.T) { + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + filebeat.RemoveAllCLIArgs() + + tempDir := filebeat.TempDir() + + // 1. Generate the log file path, but do not write data to it + logFilePath := path.Join(tempDir, "log.log") + + // 2. Create the log file + integration.GenerateLogFile(t, logFilePath, 10, false) + + // 3. Write configuration file ans start Filebeat + filebeat.WriteConfigFile(fmt.Sprintf(filebeatBasicConfig, logFilePath, tempDir)) + filebeat.Start() + + // We're not interested in data ingestion, we just want to ensure Filebeat can run + // and logs to file when no logging is explicitly configured + + // The default logs home path is `path.logs: ${path.home}/logs` + logFileName := fmt.Sprintf("filebeat-%s.ndjson", time.Now().Format("20060102")) + filebeatLogFile := filepath.Join(tempDir, "logs", logFileName) + + var f *os.File + var err error + require.Eventuallyf(t, func() bool { + fmt.Println(".") + f, err = os.Open(filebeatLogFile) + if err != nil { + return false + } + return true + }, 10*time.Second, time.Millisecond, "could not read log file '%s'", filebeatLogFile) + defer f.Close() + + r := bufio.NewScanner(f) + count := 0 + for r.Scan() { + line := r.Bytes() + m := map[string]any{} + if err := json.Unmarshal(line, &m); err != nil { + t.Fatalf("line %d is not a valid JSON: %s", count, err) + } + count++ + } +} diff --git a/go.mod b/go.mod index a7e125f8f91..42bd3ecc671 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.8 + github.com/elastic/elastic-agent-libs v0.9.9 github.com/elastic/elastic-agent-system-metrics v0.10.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.12.2 diff --git a/go.sum b/go.sum index 7ef2143875e..397e93e10ea 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.8 h1:fwl3hp0gNmKkuERcUQTwe4cyIK6M0jJkv16EIsB6Apw= -github.com/elastic/elastic-agent-libs v0.9.8/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= +github.com/elastic/elastic-agent-libs v0.9.9 h1:NwJfGKb2KwukMJvOeIxxjRQpNap546xeBFztxqzlSWg= +github.com/elastic/elastic-agent-libs v0.9.9/go.mod h1:CpBqDK8Aql/ou57UR4bWNQt0Cfr14rTRYBmWyiwDpW0= github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc= github.com/elastic/elastic-agent-system-metrics v0.10.2/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= @@ -2435,6 +2435,8 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index cc305255709..fe5a1bbfd60 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -627,6 +627,18 @@ func (b *BeatProc) LoadMeta() (Meta, error) { return m, nil } +// RemoveAllCLIArgs removes all CLI arguments configured. +// It will also remove all configuration for home path and +// logs, there fore some methods, like the ones that read logs, +// might fail if Filebeat is not configured the way this framework +// expects. +// +// The only CLI argument kept is the `--systemTest` that is necessary +// to make the System Test Binary run Filebeat +func (b *BeatProc) RemoveAllCLIArgs() { + b.baseArgs = []string{b.beatName, "--systemTest"} +} + func (b *BeatProc) Stdin() io.WriteCloser { return b.stdin } From d213f4e4c8fbf5af645f47ea7d7fcaaf1a7045b7 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 30 May 2024 09:08:50 -0400 Subject: [PATCH 2/5] Update elastic-agent-libs to v0.9.10 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index a9af6aa666f..32252249c79 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.9 +Version: v0.9.10 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.10/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 42bd3ecc671..422e0bbb691 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.9 + github.com/elastic/elastic-agent-libs v0.9.10 github.com/elastic/elastic-agent-system-metrics v0.10.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.12.2 diff --git a/go.sum b/go.sum index 397e93e10ea..5d3d8679239 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.9 h1:NwJfGKb2KwukMJvOeIxxjRQpNap546xeBFztxqzlSWg= -github.com/elastic/elastic-agent-libs v0.9.9/go.mod h1:CpBqDK8Aql/ou57UR4bWNQt0Cfr14rTRYBmWyiwDpW0= +github.com/elastic/elastic-agent-libs v0.9.10 h1:t6eM2s2EXv/AzXXdFy8eWS+/HAzE7AZL1yo3lSAPsfo= +github.com/elastic/elastic-agent-libs v0.9.10/go.mod h1:CpBqDK8Aql/ou57UR4bWNQt0Cfr14rTRYBmWyiwDpW0= github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc= github.com/elastic/elastic-agent-system-metrics v0.10.2/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= From d9dacd1fa9ac046f9476a910746f4ad14608bc1d Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 3 Jun 2024 09:26:25 -0400 Subject: [PATCH 3/5] Update elastic-agent-libs to v0.9.11 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 32252249c79..4e296959fc1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.10 +Version: v0.9.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.10/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.11/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 422e0bbb691..6620085e334 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.10 + github.com/elastic/elastic-agent-libs v0.9.11 github.com/elastic/elastic-agent-system-metrics v0.10.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.12.2 diff --git a/go.sum b/go.sum index 5d3d8679239..14d8c356747 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.10 h1:t6eM2s2EXv/AzXXdFy8eWS+/HAzE7AZL1yo3lSAPsfo= -github.com/elastic/elastic-agent-libs v0.9.10/go.mod h1:CpBqDK8Aql/ou57UR4bWNQt0Cfr14rTRYBmWyiwDpW0= +github.com/elastic/elastic-agent-libs v0.9.11 h1:J4aduNJhVeb699FxJIW/dD4BPREILqXgpWD41sCw8Uc= +github.com/elastic/elastic-agent-libs v0.9.11/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw= github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc= github.com/elastic/elastic-agent-system-metrics v0.10.2/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= From e8b7311c0aac86fd68748b649a63d8d5200b7eea Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 3 Jun 2024 09:41:48 -0400 Subject: [PATCH 4/5] Remove debug log --- filebeat/tests/integration/filebeat_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/filebeat/tests/integration/filebeat_test.go b/filebeat/tests/integration/filebeat_test.go index a88631bdbea..4a9e9cf5615 100644 --- a/filebeat/tests/integration/filebeat_test.go +++ b/filebeat/tests/integration/filebeat_test.go @@ -74,7 +74,6 @@ func TestFilebeatRunsAndLogsJSONToFile(t *testing.T) { var f *os.File var err error require.Eventuallyf(t, func() bool { - fmt.Println(".") f, err = os.Open(filebeatLogFile) if err != nil { return false From c7a4948c0d311a1792fe755eb16e90d469629a12 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 3 Jun 2024 12:16:48 -0400 Subject: [PATCH 5/5] Fix lint warnings --- filebeat/tests/integration/filebeat_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/filebeat/tests/integration/filebeat_test.go b/filebeat/tests/integration/filebeat_test.go index 4a9e9cf5615..f2c5b3dc614 100644 --- a/filebeat/tests/integration/filebeat_test.go +++ b/filebeat/tests/integration/filebeat_test.go @@ -73,12 +73,11 @@ func TestFilebeatRunsAndLogsJSONToFile(t *testing.T) { var f *os.File var err error + // We have to wait until the file is created, so we wait + // until `os.Open` returns no error. require.Eventuallyf(t, func() bool { f, err = os.Open(filebeatLogFile) - if err != nil { - return false - } - return true + return err == nil }, 10*time.Second, time.Millisecond, "could not read log file '%s'", filebeatLogFile) defer f.Close()