[main](backport #41489) Revert system module support for journald (#4…

…1490) * Revert system module support for journald (#41489) Revert the system module usage of the system-logs input that was enabling it to run the Journald input. The revert is done in the system module configuration, pipelines and documentation. The system-logs input and its tests are kept. (cherry picked from commit 00d7161) # Conflicts: # filebeat/module/system/auth/config/auth.yml # filebeat/module/system/syslog/config/syslog.yml # filebeat/tests/integration/systemlogs_linux_test.go * resolve merge conflicts * skip system-logs tests --------- Co-authored-by: Tiago Queiroz <[email protected]>
elastic · Nov 1, 2024 · 3c16c29 · 3c16c29
1 parent b1c7478
commit 3c16c29
Show file tree

Hide file tree

Showing 30 changed files with 97 additions and 898 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -48,7 +48,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099]
 - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
 - The performance of ingesting SQS data with the S3 input has improved by up to 60x for queues with many small events. `max_number_of_messages` config for SQS mode is now ignored, as the new design no longer needs a manual cap on messages. Instead, use `number_of_workers` to scale ingestion rate in both S3 and SQS modes. The increased efficiency may increase network bandwidth consumption, which can be throttled by lowering `number_of_workers`. It may also increase number of events stored in memory, which can be throttled by lowering the configured size of the internal queue. {pull}40699[40699]
-- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061]
 
 - Add kafka compression support for ZSTD.
 
@@ -324,7 +323,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add CSV decoding capacity to gcs input {pull}40979[40979]
 - Add support to source AWS cloudwatch logs from linked accounts. {pull}41188[41188]
 - Jounrald input now supports filtering by facilities {pull}41061[41061]
-- System module now supports reading from jounrald. {pull}41061[41061]
 - Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206]
 - Improved Azure Blob Storage input documentation. {pull}41252[41252]
 - Make ETW input GA. {pull}41389[41389]

diff --git a/filebeat/docs/include/use-journald.asciidoc b/filebeat/docs/include/use-journald.asciidoc
diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc
@@ -23,7 +23,7 @@ include::../include/gs-link.asciidoc[]
 === Compatibility
 
 This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
-macOS Sierra. For Debian 12 Journald is used to read the system logs.
+macOS Sierra.
 
 This module is not available for Windows.
 
@@ -65,15 +65,11 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/use-journald.asciidoc[]
-
 [float]
 ==== `auth` fileset settings
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/use-journald.asciidoc[]
-
 *`var.tags`*::
 
 A list of tags to include in events. Including `forwarded` indicates that the

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -21,18 +21,7 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Force using journald to collect system logs
-    #var.use_journald: true|false
-
-    # Force using log files to collect system logs
-    #var.use_files: true|false
-
-    # If use_journald and use_files are false, then
-    # Filebeat will autodetect whether use to journald
-    # to collect system logs.
-
-    # Input configuration (advanced).
-    # Any input configuration option
+    # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
 
@@ -44,23 +33,6 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Force using journald to collect system logs
-    #var.use_journald: true|false
-
-    # Force using log files to collect system logs
-    #var.use_files: true|false
-
-    # If use_journald and use_files are false, then
-    # Filebeat will autodetect whether use to journald
-    # to collect system logs.
-
-    # A list of tags to include in events. Including 'forwarded'
-    # indicates that the events did not originate on this host and
-    # causes host.name to not be added to events. Include
-    # 'preserve_orginal_event' causes the pipeline to retain the raw log
-    # in event.original. Defaults to [].
-    #var.tags: []
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:

diff --git a/filebeat/module/system/README.md b/filebeat/module/system/README.md
diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml
@@ -7,18 +7,7 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Force using journald to collect system logs
-    #var.use_journald: true|false
-
-    # Force using log files to collect system logs
-    #var.use_files: true|false
-
-    # If use_journald and use_files are false, then
-    # Filebeat will autodetect whether use to journald
-    # to collect system logs.
-
-    # Input configuration (advanced).
-    # Any input configuration option
+    # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
 
@@ -30,23 +19,6 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Force using journald to collect system logs
-    #var.use_journald: true|false
-
-    # Force using log files to collect system logs
-    #var.use_files: true|false
-
-    # If use_journald and use_files are false, then
-    # Filebeat will autodetect whether use to journald
-    # to collect system logs.
-
-    # A list of tags to include in events. Including 'forwarded'
-    # indicates that the events did not originate on this host and
-    # causes host.name to not be added to events. Include
-    # 'preserve_orginal_event' causes the pipeline to retain the raw log
-    # in event.original. Defaults to [].
-    #var.tags: []
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml
@@ -7,37 +7,10 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Force using journald to collect system logs
-    #var.use_journald: true|false
-
-    # Force using log files to collect system logs
-    #var.use_files: true|false
-
-    # If use_journald and use_files are false, then
-    # Filebeat will autodetect whether use to journald
-    # to collect system logs.
-
   # Authorization logs
   auth:
     enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
-
-    # Force using journald to collect system logs
-    #var.use_journald: true|false
-
-    # Force using log files to collect system logs
-    #var.use_files: true|false
-
-    # If use_journald and use_files are false, then
-    # Filebeat will autodetect whether use to journald
-    # to collect system logs.
-
-    # A list of tags to include in events. Including forwarded
-    # indicates that the events did not originate on this host and
-    # causes host.name to not be added to events. Include
-    # preserve_orginal_event causes the pipeline to retain the raw log
-    # in event.original. Defaults to [].
-    #var.tags: []
diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc
@@ -16,7 +16,7 @@ include::../include/gs-link.asciidoc[]
 === Compatibility
 
 This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
-macOS Sierra. For Debian 12 Journald is used to read the system logs.
+macOS Sierra.
 
 This module is not available for Windows.
 
@@ -58,15 +58,11 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/use-journald.asciidoc[]
-
 [float]
 ==== `auth` fileset settings
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/use-journald.asciidoc[]
-
 *`var.tags`*::
 
 A list of tags to include in events. Including `forwarded` indicates that the

diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml
@@ -1,35 +1,17 @@
-type: system-logs
-id: system-auth
-
-{{ if .use_journald }}
-use_journald: true
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
 {{ end }}
+exclude_files: [".gz$"]
 
-{{ if .use_files }}
-use_files: true
-{{ end }}
+multiline:
+  pattern: "^\\s"
+  match: after
 
-tags: {{ .tags | tojson }}
 processors:
   - add_locale: ~
 
-publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
-
-journald:
-  id: system-auth
-  facilities:
-    - 4
-    - 10
-
-files:
-  id: system-auth
-  paths:
-  {{ range $i, $path := .paths }}
-    - {{$path}}
-  {{ end }}
-  exclude_files: [".gz$"]
-
-  multiline:
-    pattern: "^\\s"
-    match: after
+tags: {{ .tags | tojson }}
 
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
diff --git a/filebeat/module/system/auth/ingest/entrypoint.yml b/filebeat/module/system/auth/ingest/entrypoint.yml
diff --git a/filebeat/module/system/auth/ingest/files.yml b/filebeat/module/system/auth/ingest/files.yml
diff --git a/filebeat/module/system/auth/ingest/grok-auth-messages.yml b/filebeat/module/system/auth/ingest/grok-auth-messages.yml
diff --git a/filebeat/module/system/auth/ingest/journald.yml b/filebeat/module/system/auth/ingest/journald.yml