From df064b8150517ff085b32f617af0be56c2b0a908 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 3 Jun 2024 15:23:04 -0400 Subject: [PATCH] update elastic-agent-libs to v0.9.11 (#39765) Update elastic-agent-libs to v0.9.11 and add tests to ensure Beats always logs as JSON to a file when no logging is explicitly configured. An integration test is added to assert the behaviour. --- NOTICE.txt | 33 +++++++- filebeat/tests/integration/filebeat_test.go | 94 +++++++++++++++++++++ go.mod | 2 +- go.sum | 6 +- libbeat/tests/integration/framework.go | 12 +++ 5 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 filebeat/tests/integration/filebeat_test.go diff --git a/NOTICE.txt b/NOTICE.txt index 4e0834b9769..4e296959fc1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.8 +Version: v0.9.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.11/LICENSE: Apache License Version 2.0, January 2004 @@ -55932,6 +55932,35 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L limitations under the License. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/mcuadros/go-syslog.v2 +Version: v2.3.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/mcuadros/go-syslog.v2@v2.3.0/LICENSE: + +Copyright (c) 2013 Máximo Cuadros + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : gopkg.in/tomb.v1 Version: v1.0.0-20141024135613-dd632973f1e7 diff --git a/filebeat/tests/integration/filebeat_test.go b/filebeat/tests/integration/filebeat_test.go new file mode 100644 index 00000000000..f2c5b3dc614 --- /dev/null +++ b/filebeat/tests/integration/filebeat_test.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package integration + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +var filebeatBasicConfig = ` +filebeat.inputs: + - type: filestream + id: "test-filebeat-can-log" + paths: + - %s +path.home: %s +output.discard.enabled: true +` + +func TestFilebeatRunsAndLogsJSONToFile(t *testing.T) { + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + filebeat.RemoveAllCLIArgs() + + tempDir := filebeat.TempDir() + + // 1. Generate the log file path, but do not write data to it + logFilePath := path.Join(tempDir, "log.log") + + // 2. Create the log file + integration.GenerateLogFile(t, logFilePath, 10, false) + + // 3. Write configuration file ans start Filebeat + filebeat.WriteConfigFile(fmt.Sprintf(filebeatBasicConfig, logFilePath, tempDir)) + filebeat.Start() + + // We're not interested in data ingestion, we just want to ensure Filebeat can run + // and logs to file when no logging is explicitly configured + + // The default logs home path is `path.logs: ${path.home}/logs` + logFileName := fmt.Sprintf("filebeat-%s.ndjson", time.Now().Format("20060102")) + filebeatLogFile := filepath.Join(tempDir, "logs", logFileName) + + var f *os.File + var err error + // We have to wait until the file is created, so we wait + // until `os.Open` returns no error. + require.Eventuallyf(t, func() bool { + f, err = os.Open(filebeatLogFile) + return err == nil + }, 10*time.Second, time.Millisecond, "could not read log file '%s'", filebeatLogFile) + defer f.Close() + + r := bufio.NewScanner(f) + count := 0 + for r.Scan() { + line := r.Bytes() + m := map[string]any{} + if err := json.Unmarshal(line, &m); err != nil { + t.Fatalf("line %d is not a valid JSON: %s", count, err) + } + count++ + } +} diff --git a/go.mod b/go.mod index a7e125f8f91..6620085e334 100644 --- a/go.mod +++ b/go.mod @@ -206,7 +206,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.6.14 - github.com/elastic/elastic-agent-libs v0.9.8 + github.com/elastic/elastic-agent-libs v0.9.11 github.com/elastic/elastic-agent-system-metrics v0.10.2 github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/elastic/mito v1.12.2 diff --git a/go.sum b/go.sum index 7ef2143875e..14d8c356747 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE= github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.8 h1:fwl3hp0gNmKkuERcUQTwe4cyIK6M0jJkv16EIsB6Apw= -github.com/elastic/elastic-agent-libs v0.9.8/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= +github.com/elastic/elastic-agent-libs v0.9.11 h1:J4aduNJhVeb699FxJIW/dD4BPREILqXgpWD41sCw8Uc= +github.com/elastic/elastic-agent-libs v0.9.11/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw= github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc= github.com/elastic/elastic-agent-system-metrics v0.10.2/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI= github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= @@ -2435,6 +2435,8 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index cc305255709..fe5a1bbfd60 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -627,6 +627,18 @@ func (b *BeatProc) LoadMeta() (Meta, error) { return m, nil } +// RemoveAllCLIArgs removes all CLI arguments configured. +// It will also remove all configuration for home path and +// logs, there fore some methods, like the ones that read logs, +// might fail if Filebeat is not configured the way this framework +// expects. +// +// The only CLI argument kept is the `--systemTest` that is necessary +// to make the System Test Binary run Filebeat +func (b *BeatProc) RemoveAllCLIArgs() { + b.baseArgs = []string{b.beatName, "--systemTest"} +} + func (b *BeatProc) Stdin() io.WriteCloser { return b.stdin }