diff --git a/filebeat/input/journald/input_filtering_test.go b/filebeat/input/journald/input_filtering_test.go index 1aa58d1f8bc3..220f71e2d9ba 100644 --- a/filebeat/input/journald/input_filtering_test.go +++ b/filebeat/input/journald/input_filtering_test.go @@ -22,7 +22,7 @@ package journald import ( "context" "encoding/json" - "path" + "path/filepath" "testing" "time" @@ -30,6 +30,8 @@ import ( ) func TestInputSyslogIdentifier(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "input-multiline-parser.journal.gz")) + tests := map[string]struct { identifiers []string expectedMessages []string @@ -53,7 +55,7 @@ func TestInputSyslogIdentifier(t *testing.T) { t.Run(name, func(t *testing.T) { env := newInputTestingEnvironment(t) inp := env.mustCreateInput(mapstr.M{ - "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "paths": []string{out}, "syslog_identifiers": testCase.identifiers, }) @@ -73,6 +75,7 @@ func TestInputSyslogIdentifier(t *testing.T) { } func TestInputUnits(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "input-multiline-parser.journal.gz")) tests := map[string]struct { units []string kernel bool @@ -110,7 +113,7 @@ func TestInputUnits(t *testing.T) { t.Run(name, func(t *testing.T) { env := newInputTestingEnvironment(t) inp := env.mustCreateInput(mapstr.M{ - "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "paths": []string{out}, "units": testCase.units, "kernel": testCase.kernel, }) @@ -131,6 +134,7 @@ func TestInputUnits(t *testing.T) { } func TestInputIncludeMatches(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "input-multiline-parser.journal.gz")) tests := map[string]struct { includeMatches map[string]interface{} expectedMessages []string @@ -168,7 +172,7 @@ func TestInputIncludeMatches(t *testing.T) { t.Run(name, func(t *testing.T) { env := newInputTestingEnvironment(t) inp := env.mustCreateInput(mapstr.M{ - "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "paths": []string{out}, "include_matches": testCase.includeMatches, }) @@ -190,6 +194,7 @@ func TestInputIncludeMatches(t *testing.T) { // TestInputSeek test the output of various seek modes while reading // from input-multiline-parser.journal. func TestInputSeek(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "input-multiline-parser.journal.gz")) // Uncomment the following line to see all logs during the test execution // logp.DevelopmentSetup() timeAfterFirstEvent := time.Date(2021, time.November, 22, 17, 10, 20, 0, time.UTC).In(time.Local) @@ -256,13 +261,13 @@ func TestInputSeek(t *testing.T) { if err := json.Unmarshal([]byte(testCase.cursor), &tmp); err != nil { t.Fatal(err) } - if err := store.Set("journald::testdata/input-multiline-parser.journal", tmp); err != nil { + if err := store.Set("journald::"+out, tmp); err != nil { t.Fatal(err) } } conf := mapstr.M{ - "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "paths": []string{out}, } conf.DeepUpdate(testCase.config) diff --git a/filebeat/input/journald/input_parsers_test.go b/filebeat/input/journald/input_parsers_test.go index c1c2c6f6bb5b..35df3cc1e848 100644 --- a/filebeat/input/journald/input_parsers_test.go +++ b/filebeat/input/journald/input_parsers_test.go @@ -21,7 +21,7 @@ package journald import ( "context" - "path" + "path/filepath" "testing" "github.com/elastic/elastic-agent-libs/mapstr" @@ -31,9 +31,11 @@ import ( // it only tests a single parser, but that is enough to ensure // we're correctly using the parsers func TestInputParsers(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "ndjson-parser.journal.gz")) + env := newInputTestingEnvironment(t) inp := env.mustCreateInput(mapstr.M{ - "paths": []string{path.Join("testdata", "ndjson-parser.journal")}, + "paths": []string{out}, "parsers": []mapstr.M{ { "ndjson": mapstr.M{ diff --git a/filebeat/input/journald/input_test.go b/filebeat/input/journald/input_test.go index b82663c52626..776115d5d8ac 100644 --- a/filebeat/input/journald/input_test.go +++ b/filebeat/input/journald/input_test.go @@ -20,13 +20,15 @@ package journald import ( + "compress/gzip" "context" _ "embed" "encoding/json" "fmt" + "io" "os" - "path" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -40,9 +42,11 @@ import ( ) func TestInputCanReadAllBoots(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "multiple-boots.journal.gz")) + env := newInputTestingEnvironment(t) cfg := mapstr.M{ - "paths": []string{path.Join("testdata", "multiple-boots.journal")}, + "paths": []string{out}, } inp := env.mustCreateInput(cfg) @@ -54,6 +58,7 @@ func TestInputCanReadAllBoots(t *testing.T) { } func TestInputFieldsTranslation(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "input-multiline-parser.journal.gz")) // A few random keys to verify keysToCheck := map[string]string{ "systemd.user_unit": "log-service.service", @@ -74,7 +79,7 @@ func TestInputFieldsTranslation(t *testing.T) { env := newInputTestingEnvironment(t) inp := env.mustCreateInput(mapstr.M{ - "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "paths": []string{out}, "include_matches.match": []string{"_SYSTEMD_USER_UNIT=log-service.service"}, "save_remote_hostname": tc.saveRemoteHostname, }) @@ -118,9 +123,10 @@ func TestInputFieldsTranslation(t *testing.T) { // __CURSOR - it is added to the registry and there are other tests for it // __MONOTONIC_TIMESTAMP - it is part of the cursor func TestCompareGoSystemdWithJournalctl(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "input-multiline-parser.journal.gz")) env := newInputTestingEnvironment(t) inp := env.mustCreateInput(mapstr.M{ - "paths": []string{path.Join("testdata", "input-multiline-parser.journal")}, + "paths": []string{out}, "seek": "head", }) @@ -185,6 +191,7 @@ func TestCompareGoSystemdWithJournalctl(t *testing.T) { } func TestMatchers(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "matchers.journal.gz")) // If this test fails, uncomment the following line to see the debug logs // logp.DevelopmentSetup() testCases := []struct { @@ -274,7 +281,7 @@ func TestMatchers(t *testing.T) { t.Run(tc.name, func(t *testing.T) { env := newInputTestingEnvironment(t) cfg := mapstr.M{ - "paths": []string{path.Join("testdata", "matchers.journal")}, + "paths": []string{out}, "include_matches": tc.matchers, } cfg.Update(mapstr.M(tc.confiFields)) @@ -322,3 +329,27 @@ func TestReaderAdapterCanHandleNonStringFields(t *testing.T) { }) } } + +func decompress(t *testing.T, namegz string) string { + t.Helper() + + ingz, err := os.Open(namegz) + require.NoError(t, err) + defer ingz.Close() + + out := filepath.Join(t.TempDir(), strings.TrimSuffix(filepath.Base(namegz), ".gz")) + + dst, err := os.Create(out) + require.NoError(t, err) + defer dst.Close() + + gr, err := gzip.NewReader(ingz) + require.NoError(t, err) + defer gr.Close() + + //nolint:gosec // this is used in tests + _, err = io.Copy(dst, gr) + require.NoError(t, err) + + return out +} diff --git a/filebeat/input/journald/testdata/input-multiline-parser.journal b/filebeat/input/journald/testdata/input-multiline-parser.journal deleted file mode 100644 index 9aecfd442f0c..000000000000 Binary files a/filebeat/input/journald/testdata/input-multiline-parser.journal and /dev/null differ diff --git a/filebeat/input/journald/testdata/input-multiline-parser.journal.gz b/filebeat/input/journald/testdata/input-multiline-parser.journal.gz new file mode 100644 index 000000000000..05b7d19adf39 Binary files /dev/null and b/filebeat/input/journald/testdata/input-multiline-parser.journal.gz differ diff --git a/filebeat/input/journald/testdata/matchers.journal b/filebeat/input/journald/testdata/matchers.journal deleted file mode 100644 index 06b2c24a3b85..000000000000 Binary files a/filebeat/input/journald/testdata/matchers.journal and /dev/null differ diff --git a/filebeat/input/journald/testdata/matchers.journal.gz b/filebeat/input/journald/testdata/matchers.journal.gz new file mode 100644 index 000000000000..16188698692f Binary files /dev/null and b/filebeat/input/journald/testdata/matchers.journal.gz differ diff --git a/filebeat/input/journald/testdata/multiple-boots.journal b/filebeat/input/journald/testdata/multiple-boots.journal deleted file mode 100644 index 668b82162d6a..000000000000 Binary files a/filebeat/input/journald/testdata/multiple-boots.journal and /dev/null differ diff --git a/filebeat/input/journald/testdata/multiple-boots.journal.gz b/filebeat/input/journald/testdata/multiple-boots.journal.gz new file mode 100644 index 000000000000..02d46588ada0 Binary files /dev/null and b/filebeat/input/journald/testdata/multiple-boots.journal.gz differ diff --git a/filebeat/input/journald/testdata/ndjson-parser.journal b/filebeat/input/journald/testdata/ndjson-parser.journal deleted file mode 100644 index aa4aa7960f3a..000000000000 Binary files a/filebeat/input/journald/testdata/ndjson-parser.journal and /dev/null differ diff --git a/filebeat/input/journald/testdata/ndjson-parser.journal.gz b/filebeat/input/journald/testdata/ndjson-parser.journal.gz new file mode 100644 index 000000000000..f5eb86364da5 Binary files /dev/null and b/filebeat/input/journald/testdata/ndjson-parser.journal.gz differ