diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 797db99b7ca..673af8be06e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -48,7 +48,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099] - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089] - The performance of ingesting SQS data with the S3 input has improved by up to 60x for queues with many small events. `max_number_of_messages` config for SQS mode is now ignored, as the new design no longer needs a manual cap on messages. Instead, use `number_of_workers` to scale ingestion rate in both S3 and SQS modes. The increased efficiency may increase network bandwidth consumption, which can be throttled by lowering `number_of_workers`. It may also increase number of events stored in memory, which can be throttled by lowering the configured size of the internal queue. {pull}40699[40699] -- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061] - Add kafka compression support for ZSTD. @@ -324,7 +323,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add CSV decoding capacity to gcs input {pull}40979[40979] - Add support to source AWS cloudwatch logs from linked accounts. {pull}41188[41188] - Jounrald input now supports filtering by facilities {pull}41061[41061] -- System module now supports reading from jounrald. {pull}41061[41061] - Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206] - Improved Azure Blob Storage input documentation. {pull}41252[41252] - Make ETW input GA. {pull}41389[41389] diff --git a/filebeat/docs/include/use-journald.asciidoc b/filebeat/docs/include/use-journald.asciidoc deleted file mode 100644 index 12cb33c0c6c..00000000000 --- a/filebeat/docs/include/use-journald.asciidoc +++ /dev/null @@ -1,12 +0,0 @@ -*`var.use_journald`*:: - -A boolean that when set to `true` will read logs from Journald. When -Journald is used all events contain the tag `journald` - -*`var.use_files`*:: - -A boolean that when set to `true` will read logs from the log files -defined by `vars.paths`. - -If neither `var.use_journald` nor `var.use_files` are set (or both are -`false`) {beatname_uc} will auto-detect the source for the logs. diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc index 88cb1f78a1c..1866f2d5c25 100644 --- a/filebeat/docs/modules/system.asciidoc +++ b/filebeat/docs/modules/system.asciidoc @@ -23,7 +23,7 @@ include::../include/gs-link.asciidoc[] === Compatibility This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and -macOS Sierra. For Debian 12 Journald is used to read the system logs. +macOS Sierra. This module is not available for Windows. @@ -65,15 +65,11 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] -include::../include/use-journald.asciidoc[] - [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] -include::../include/use-journald.asciidoc[] - *`var.tags`*:: A list of tags to include in events. Including `forwarded` indicates that the diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index a1af7b861d5..14e9f276fb4 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -21,18 +21,7 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # Input configuration (advanced). - # Any input configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -44,23 +33,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # A list of tags to include in events. Including 'forwarded' - # indicates that the events did not originate on this host and - # causes host.name to not be added to events. Include - # 'preserve_orginal_event' causes the pipeline to retain the raw log - # in event.original. Defaults to []. - #var.tags: [] - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/README.md b/filebeat/module/system/README.md deleted file mode 100644 index 2471264cfcf..00000000000 --- a/filebeat/module/system/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Journald tests (Debian 12) -The tests for the journald input (currently only used for Debian 12 -testing) require journal files (test files ending in `.journal`), those -files are generated using `systemd-journal-remote` (see the [Journald -input README.md](../../input/journald/README.md) for more details). - -The source for those journal files are the `.export` files in the test -folder. Those files are the raw output of `journalctl -o export`. They -are added here because journal files format change with different -versions of journald, which can cause `journalclt` to fail reading -them, which leads to test failures. So if tests start failing because -`journalctl` cannot read the journal files as expected, new ones can -easily be generated with the same version of journalctl used on CI -and the original dataset. diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml index 04160dfb1bf..3c7a0b43d49 100644 --- a/filebeat/module/system/_meta/config.reference.yml +++ b/filebeat/module/system/_meta/config.reference.yml @@ -7,18 +7,7 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # Input configuration (advanced). - # Any input configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -30,23 +19,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # A list of tags to include in events. Including 'forwarded' - # indicates that the events did not originate on this host and - # causes host.name to not be added to events. Include - # 'preserve_orginal_event' causes the pipeline to retain the raw log - # in event.original. Defaults to []. - #var.tags: [] - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml index f95f3e5969d..c1fe882374d 100644 --- a/filebeat/module/system/_meta/config.yml +++ b/filebeat/module/system/_meta/config.yml @@ -7,16 +7,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - # Authorization logs auth: enabled: false @@ -24,20 +14,3 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # A list of tags to include in events. Including forwarded - # indicates that the events did not originate on this host and - # causes host.name to not be added to events. Include - # preserve_orginal_event causes the pipeline to retain the raw log - # in event.original. Defaults to []. - #var.tags: [] diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc index 1aaca678963..6d9209eafe2 100644 --- a/filebeat/module/system/_meta/docs.asciidoc +++ b/filebeat/module/system/_meta/docs.asciidoc @@ -16,7 +16,7 @@ include::../include/gs-link.asciidoc[] === Compatibility This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and -macOS Sierra. For Debian 12 Journald is used to read the system logs. +macOS Sierra. This module is not available for Windows. @@ -58,15 +58,11 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] -include::../include/use-journald.asciidoc[] - [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] -include::../include/use-journald.asciidoc[] - *`var.tags`*:: A list of tags to include in events. Including `forwarded` indicates that the diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml index a552ae3a089..466b55078af 100644 --- a/filebeat/module/system/auth/config/auth.yml +++ b/filebeat/module/system/auth/config/auth.yml @@ -1,35 +1,17 @@ -type: system-logs -id: system-auth - -{{ if .use_journald }} -use_journald: true +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} {{ end }} +exclude_files: [".gz$"] -{{ if .use_files }} -use_files: true -{{ end }} +multiline: + pattern: "^\\s" + match: after -tags: {{ .tags | tojson }} processors: - add_locale: ~ -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -journald: - id: system-auth - facilities: - - 4 - - 10 - -files: - id: system-auth - paths: - {{ range $i, $path := .paths }} - - {{$path}} - {{ end }} - exclude_files: [".gz$"] - - multiline: - pattern: "^\\s" - match: after +tags: {{ .tags | tojson }} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/filebeat/module/system/auth/ingest/entrypoint.yml b/filebeat/module/system/auth/ingest/entrypoint.yml deleted file mode 100644 index 7da5fc4a5d4..00000000000 --- a/filebeat/module/system/auth/ingest/entrypoint.yml +++ /dev/null @@ -1,18 +0,0 @@ -description: Entrypoint Pipeline for system/auth Filebeat module -processors: - - set: - field: event.ingested - copy_from: _ingest.timestamp - - script: - source: | - if(ctx?.journald != null){ - ctx['auth_pipeline'] = '{< IngestPipeline "journald" >}'; - return; - } - ctx['auth_pipeline'] = '{< IngestPipeline "files" >}'; - return; - - pipeline: - name: "{{ auth_pipeline }}" - - remove: - ignore_failure: true - field: "auth_pipeline" diff --git a/filebeat/module/system/auth/ingest/files.yml b/filebeat/module/system/auth/ingest/files.yml deleted file mode 100644 index 557747b6400..00000000000 --- a/filebeat/module/system/auth/ingest/files.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -description: Pipeline for parsing system authorization and secure logs. -processors: - - rename: - if: ctx.event?.original == null - field: message - target_field: event.original - ignore_missing: true - - grok: - description: Grok the message header. - tag: grok-message-header - field: event.original - pattern_definitions: - GREEDYMULTILINE: '(.|\n)*' - TIMESTAMP: (?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP}) - patterns: - - '^%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:%{SPACE}%{GREEDYMULTILINE:_temp.message}$' - - pipeline: - description: Grok specific auth messages. - name: '{< IngestPipeline "grok-auth-messages" >}' - on_failure: - - rename: - description: Leave the unmatched content in message. - field: _temp.message - target_field: message - - remove: - field: _temp - - pipeline: - name: "{< IngestPipeline "common" >}" - - date: - if: ctx.event?.timezone == null - field: system.auth.timestamp - target_field: '@timestamp' - formats: - - MMM d HH:mm:ss - - MMM dd HH:mm:ss - - ISO8601 - on_failure: - - append: - field: error.message - value: '{{{ _ingest.on_failure_message }}}' - - date: - if: ctx.event?.timezone != null - field: system.auth.timestamp - target_field: '@timestamp' - formats: - - MMM d HH:mm:ss - - MMM dd HH:mm:ss - - ISO8601 - timezone: '{{{ event.timezone }}}' - on_failure: - - append: - field: error.message - value: '{{{ _ingest.on_failure_message }}}' - - remove: - field: system.auth.timestamp - - set: - field: input.type - value: log -on_failure: - - set: - field: error.message - value: '{{{ _ingest.on_failure_message }}}' diff --git a/filebeat/module/system/auth/ingest/grok-auth-messages.yml b/filebeat/module/system/auth/ingest/grok-auth-messages.yml deleted file mode 100644 index fc09abbff5e..00000000000 --- a/filebeat/module/system/auth/ingest/grok-auth-messages.yml +++ /dev/null @@ -1,14 +0,0 @@ -description: Journald Pipeline for system/auth Filebeat module -processors: - - grok: - description: Grok specific auth messages. - tag: grok-specific-messages - field: _temp.message - ignore_missing: true - patterns: - - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?' - - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}' - - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}' - - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}' - - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}' - - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$' diff --git a/filebeat/module/system/auth/ingest/journald.yml b/filebeat/module/system/auth/ingest/journald.yml deleted file mode 100644 index bb43dd63cf5..00000000000 --- a/filebeat/module/system/auth/ingest/journald.yml +++ /dev/null @@ -1,33 +0,0 @@ -description: Journald Pipeline for system/auth Filebeat module -processors: - - rename: - field: "journald.process.name" - target_field: process.name - - rename: - field: message - target_field: _temp.message - - pipeline: - description: Grok specific auth messages. - name: '{< IngestPipeline "grok-auth-messages" >}' - ignore_failure: true - - rename: - field: _temp.message - target_field: message - - pipeline: - name: "{< IngestPipeline "common" >}" - - remove: - description: Remove the extra fields added by the Journald input - ignore_missing: true - field: - - journald - - process.thread - - syslog - - systemd - - message_id - - set: - field: input.type - value: journald -on_failure: - - set: - field: error.message - value: '{{{ _ingest.on_failure_message }}}' diff --git a/filebeat/module/system/auth/ingest/common.yml b/filebeat/module/system/auth/ingest/pipeline.yml similarity index 67% rename from filebeat/module/system/auth/ingest/common.yml rename to filebeat/module/system/auth/ingest/pipeline.yml index 75c2a8e46a9..c89ef94b28a 100644 --- a/filebeat/module/system/auth/ingest/common.yml +++ b/filebeat/module/system/auth/ingest/pipeline.yml @@ -1,5 +1,42 @@ -description: Common steps for Journald and log files from system/auth Filebeat module +--- +description: Pipeline for parsing system authorization and secure logs. processors: + - set: + field: event.ingested + copy_from: _ingest.timestamp + - rename: + if: ctx.event?.original == null + field: message + target_field: event.original + ignore_missing: true + - grok: + description: Grok the message header. + tag: grok-message-header + field: event.original + pattern_definitions: + GREEDYMULTILINE: '(.|\n)*' + TIMESTAMP: (?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP}) + patterns: + - '^%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:%{SPACE}%{GREEDYMULTILINE:_temp.message}$' + - grok: + description: Grok specific auth messages. + tag: grok-specific-messages + field: _temp.message + ignore_missing: true + patterns: + - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?' + - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}' + - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}' + - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}' + - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}' + - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$' + on_failure: + - rename: + description: Leave the unmatched content in message. + field: _temp.message + target_field: message + - remove: + field: _temp - grok: description: Grok usernames from PAM messages. tag: grok-pam-users @@ -70,6 +107,33 @@ processors: on_failure: - remove: field: system.auth.ssh.dropped_ip + - date: + if: ctx.event?.timezone == null + field: system.auth.timestamp + target_field: '@timestamp' + formats: + - MMM d HH:mm:ss + - MMM dd HH:mm:ss + - ISO8601 + on_failure: + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' + - date: + if: ctx.event?.timezone != null + field: system.auth.timestamp + target_field: '@timestamp' + formats: + - MMM d HH:mm:ss + - MMM dd HH:mm:ss + - ISO8601 + timezone: '{{{ event.timezone }}}' + on_failure: + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' + - remove: + field: system.auth.timestamp - geoip: field: source.ip target_field: source.geo @@ -170,3 +234,7 @@ processors: if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml index fefc51a88a4..bf1a3623cf1 100644 --- a/filebeat/module/system/auth/manifest.yml +++ b/filebeat/module/system/auth/manifest.yml @@ -12,16 +12,6 @@ var: os.windows: [] - name: tags default: [] - - name: use_journald - default: false - - name: use_files - default: false - -ingest_pipeline: - - ingest/entrypoint.yml - - ingest/files.yml - - ingest/journald.yml - - ingest/grok-auth-messages.yml - - ingest/common.yml +ingest_pipeline: ingest/pipeline.yml input: config/auth.yml diff --git a/filebeat/module/system/auth/test/debian-12.export b/filebeat/module/system/auth/test/debian-12.export deleted file mode 100644 index 583416f6c7b..00000000000 Binary files a/filebeat/module/system/auth/test/debian-12.export and /dev/null differ diff --git a/filebeat/module/system/auth/test/debian-12.journal b/filebeat/module/system/auth/test/debian-12.journal deleted file mode 100644 index 3195198e604..00000000000 Binary files a/filebeat/module/system/auth/test/debian-12.journal and /dev/null differ diff --git a/filebeat/module/system/auth/test/debian-12.journal-expected.json b/filebeat/module/system/auth/test/debian-12.journal-expected.json deleted file mode 100644 index 2ef69b76b22..00000000000 --- a/filebeat/module/system/auth/test/debian-12.journal-expected.json +++ /dev/null @@ -1,383 +0,0 @@ -[ - { - "event.action": "ssh_login", - "event.category": [ - "authentication", - "session" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "success", - "event.timezone": "-02:00", - "event.type": [ - "info" - ], - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Accepted publickey for vagrant from 10.0.2.2 port 48274 ssh2: ED25519 SHA256:k1kjhwoH/H3w31MbGOIGd7qxrkSQJnoAN0eYJVHDmmI", - "process.args": [ - "\"sshd: vagrant [priv]\"" - ], - "process.args_count": 1, - "process.command_line": "\"sshd: vagrant [priv]\"", - "process.name": "sshd", - "process.pid": 26538, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.ip": [ - "10.0.2.2" - ], - "related.user": [ - "vagrant" - ], - "service.type": "system", - "source.address": "10.0.2.2", - "source.ip": "10.0.2.2", - "source.port": 48274, - "system.auth.ssh.event": "Accepted", - "system.auth.ssh.method": "publickey", - "system.auth.ssh.signature": "ED25519 SHA256:k1kjhwoH/H3w31MbGOIGd7qxrkSQJnoAN0eYJVHDmmI", - "user.group.id": "0", - "user.id": "0", - "user.name": "vagrant" - }, - { - "event.action": "ssh_login", - "event.category": [ - "authentication", - "session" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "success", - "event.timezone": "-02:00", - "event.type": [ - "info" - ], - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Accepted password for vagrant from 192.168.42.119 port 55310 ssh2", - "process.args": [ - "\"sshd: vagrant [priv]\"" - ], - "process.args_count": 1, - "process.command_line": "\"sshd: vagrant [priv]\"", - "process.name": "sshd", - "process.pid": 1710, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.ip": [ - "192.168.42.119" - ], - "related.user": [ - "vagrant" - ], - "service.type": "system", - "source.address": "192.168.42.119", - "source.ip": "192.168.42.119", - "source.port": 55310, - "system.auth.ssh.event": "Accepted", - "system.auth.ssh.method": "password", - "user.group.id": "0", - "user.id": "0", - "user.name": "vagrant" - }, - { - "event.action": "ssh_login", - "event.category": [ - "authentication" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "failure", - "event.timezone": "-02:00", - "event.type": [ - "info" - ], - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Invalid user test from 192.168.42.119 port 48890", - "process.args": [ - "\"sshd: unknown [priv]\"" - ], - "process.args_count": 1, - "process.command_line": "\"sshd: unknown [priv]\"", - "process.name": "sshd", - "process.pid": 1721, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.ip": [ - "192.168.42.119" - ], - "related.user": [ - "test" - ], - "service.type": "system", - "source.address": "192.168.42.119", - "source.ip": "192.168.42.119", - "system.auth.ssh.event": "Invalid", - "user.group.id": "0", - "user.id": "0", - "user.name": "test" - }, - { - "event.action": "ssh_login", - "event.category": [ - "authentication" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "failure", - "event.timezone": "-02:00", - "event.type": [ - "info" - ], - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", - "process.args": [ - "\"sshd: root [priv]\"" - ], - "process.args_count": 1, - "process.command_line": "\"sshd: root [priv]\"", - "process.name": "sshd", - "process.pid": 1723, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.ip": [ - "192.168.42.119" - ], - "related.user": [ - "root" - ], - "service.type": "system", - "source.address": "192.168.42.119", - "source.ip": "192.168.42.119", - "source.port": 46632, - "system.auth.ssh.event": "Failed", - "system.auth.ssh.method": "password", - "user.group.id": "0", - "user.id": "0", - "user.name": "root" - }, - { - "event.action": "ssh_login", - "event.category": [ - "authentication" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "failure", - "event.timezone": "-02:00", - "event.type": [ - "info" - ], - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", - "process.args": [ - "\"sshd: root [priv]\"" - ], - "process.args_count": 1, - "process.command_line": "\"sshd: root [priv]\"", - "process.name": "sshd", - "process.pid": 1723, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.ip": [ - "192.168.42.119" - ], - "related.user": [ - "root" - ], - "service.type": "system", - "source.address": "192.168.42.119", - "source.ip": "192.168.42.119", - "source.port": 46632, - "system.auth.ssh.event": "Failed", - "system.auth.ssh.method": "password", - "user.group.id": "0", - "user.id": "0", - "user.name": "root" - }, - { - "event.action": "ssh_login", - "event.category": [ - "authentication" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "failure", - "event.timezone": "-02:00", - "event.type": [ - "info" - ], - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", - "process.args": [ - "\"sshd: root [priv]\"" - ], - "process.args_count": 1, - "process.command_line": "\"sshd: root [priv]\"", - "process.name": "sshd", - "process.pid": 1723, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.ip": [ - "192.168.42.119" - ], - "related.user": [ - "root" - ], - "service.type": "system", - "source.address": "192.168.42.119", - "source.ip": "192.168.42.119", - "source.port": 46632, - "system.auth.ssh.event": "Failed", - "system.auth.ssh.method": "password", - "user.group.id": "0", - "user.id": "0", - "user.name": "root" - }, - { - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.timezone": "-02:00", - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 10, - "log.syslog.priority": 5, - "message": " vagrant : TTY=pts/2 ; PWD=/home/vagrant ; USER=root ; COMMAND=/usr/bin/emacs /etc/ssh/sshd_config", - "process.args": [ - "sudo", - "emacs", - "/etc/ssh/sshd_config" - ], - "process.args_count": 3, - "process.command_line": "sudo emacs /etc/ssh/sshd_config", - "process.name": "sudo", - "process.pid": 1582, - "related.hosts": [ - "vagrant-debian-12" - ], - "related.user": [ - " vagrant", - "root" - ], - "service.type": "system", - "system.auth.sudo.command": "/usr/bin/emacs /etc/ssh/sshd_config", - "system.auth.sudo.pwd": "/home/vagrant", - "system.auth.sudo.tty": "pts/2", - "system.auth.sudo.user": "root", - "user.effective.name": "root", - "user.group.id": "1000", - "user.id": "1000", - "user.name": " vagrant" - }, - { - "event.category": [ - "iam" - ], - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.outcome": "success", - "event.timezone": "-02:00", - "event.type": [ - "creation", - "group" - ], - "fileset.name": "auth", - "group.id": "1001", - "group.name": "test", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 10, - "log.syslog.priority": 6, - "message": "new group: name=test, GID=1001", - "process.args": [ - "/sbin/groupadd", - "-g", - "1001", - "test" - ], - "process.args_count": 4, - "process.command_line": "/sbin/groupadd -g 1001 test", - "process.name": "groupadd", - "process.pid": 1743, - "related.hosts": [ - "vagrant-debian-12" - ], - "service.type": "system", - "user.effective.group.id": "0", - "user.effective.id": "0", - "user.id": "1000" - }, - { - "event.dataset": "system.auth", - "event.kind": "event", - "event.module": "system", - "event.timezone": "-02:00", - "fileset.name": "auth", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 4, - "log.syslog.priority": 6, - "message": "Session 8 logged out. Waiting for processes to exit.", - "process.args": [ - "/lib/systemd/systemd-logind" - ], - "process.args_count": 1, - "process.command_line": "/lib/systemd/systemd-logind", - "process.name": "systemd-logind", - "process.pid": 316, - "related.hosts": [ - "vagrant-debian-12" - ], - "service.type": "system", - "user.group.id": "0", - "user.id": "0" - } -] \ No newline at end of file diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml index 6a269b314d3..e7f238d8af8 100644 --- a/filebeat/module/system/syslog/config/syslog.yml +++ b/filebeat/module/system/syslog/config/syslog.yml @@ -1,45 +1,15 @@ -type: system-logs -id: system-syslog - -{{ if .use_journald }} -use_journald: true +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} {{ end }} - -{{ if .use_files }} -use_files: true -{{ end }} - +exclude_files: [".gz$"] +multiline: + pattern: "^\\s" + match: after processors: - add_locale: ~ - add_fields: target: '' fields: ecs.version: 1.12.0 - -journald: - id: system-syslog - facilities: - - 0 - - 1 - - 2 - - 3 - - 5 - - 6 - - 7 - - 8 - - 9 - - 11 - - 12 - - 15 - -files: - id: system-syslog - paths: - {{ range $i, $path := .paths }} - - {{$path}} - {{ end }} - - exclude_files: [".gz$"] - multiline: - pattern: "^\\s" - match: after diff --git a/filebeat/module/system/syslog/ingest/entrypoint.yml b/filebeat/module/system/syslog/ingest/entrypoint.yml deleted file mode 100644 index e9f3fbc3977..00000000000 --- a/filebeat/module/system/syslog/ingest/entrypoint.yml +++ /dev/null @@ -1,15 +0,0 @@ -description: Entrypoint Pipeline for system/syslog Filebeat module -processors: - - script: - source: | - if(ctx?.journald != null){ - ctx['syslog_pipeline'] = '{< IngestPipeline "journald" >}'; - return; - } - ctx['syslog_pipeline'] = '{< IngestPipeline "files" >}'; - return; - - pipeline: - name: "{{ syslog_pipeline }}" - - remove: - ignore_failure: true - field: "syslog_pipeline" diff --git a/filebeat/module/system/syslog/ingest/journald.yml b/filebeat/module/system/syslog/ingest/journald.yml deleted file mode 100644 index 30952e96aea..00000000000 --- a/filebeat/module/system/syslog/ingest/journald.yml +++ /dev/null @@ -1,36 +0,0 @@ -description: Journald Pipeline for system/syslog Filebeat module -processors: -- set: - field: event.ingested - copy_from: _ingest.timestamp -- set: - field: "process.pid" - value: "{{ journald.pid }}" -- set: - field: "process.name" - value: "{{ journald.process.name }}" - ignore_failure: true -- set: - field: event.kind - value: event -- append: - field: related.hosts - value: "{{host.hostname}}" - if: "ctx.host?.hostname != null && ctx.host?.hostname != ''" - allow_duplicates: false -- remove: - description: Remove the extra fields added by the Journald input - ignore_missing: true - field: - - journald - - process.thread - - syslog - - systemd - - message_id -- set: - field: input.type - value: journald -on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/system/syslog/ingest/files.yml b/filebeat/module/system/syslog/ingest/pipeline.yml similarity index 97% rename from filebeat/module/system/syslog/ingest/files.yml rename to filebeat/module/system/syslog/ingest/pipeline.yml index f4f5930e198..b1352f2ad62 100644 --- a/filebeat/module/system/syslog/ingest/files.yml +++ b/filebeat/module/system/syslog/ingest/pipeline.yml @@ -59,9 +59,6 @@ processors: value: "{{host.hostname}}" if: "ctx.host?.hostname != null && ctx.host?.hostname != ''" allow_duplicates: false -- set: - field: input.type - value: log on_failure: - set: field: error.message diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml index 5112ddc5c15..39a34e56ca3 100644 --- a/filebeat/module/system/syslog/manifest.yml +++ b/filebeat/module/system/syslog/manifest.yml @@ -8,14 +8,6 @@ var: os.darwin: - /var/log/system.log* os.windows: [] - - name: use_journald - default: false - - name: use_files - default: false - -ingest_pipeline: - - ingest/entrypoint.yml - - ingest/files.yml - - ingest/journald.yml +ingest_pipeline: ingest/pipeline.yml input: config/syslog.yml diff --git a/filebeat/module/system/syslog/test/debian-12.export b/filebeat/module/system/syslog/test/debian-12.export deleted file mode 100644 index 780bd46990e..00000000000 Binary files a/filebeat/module/system/syslog/test/debian-12.export and /dev/null differ diff --git a/filebeat/module/system/syslog/test/debian-12.journal b/filebeat/module/system/syslog/test/debian-12.journal deleted file mode 100644 index f4c01a22c3f..00000000000 Binary files a/filebeat/module/system/syslog/test/debian-12.journal and /dev/null differ diff --git a/filebeat/module/system/syslog/test/debian-12.journal-expected.json b/filebeat/module/system/syslog/test/debian-12.journal-expected.json deleted file mode 100644 index 294550fb2db..00000000000 --- a/filebeat/module/system/syslog/test/debian-12.journal-expected.json +++ /dev/null @@ -1,66 +0,0 @@ -[ - { - "event.dataset": "system.syslog", - "event.kind": "event", - "event.module": "system", - "event.timezone": "-02:00", - "fileset.name": "syslog", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 3, - "log.syslog.priority": 6, - "message": "Stopped target getty.target - Login Prompts.", - "process.args": [ - "/sbin/init" - ], - "process.args_count": 1, - "process.command_line": "/sbin/init", - "process.name": "systemd", - "process.pid": "1", - "related.hosts": [ - "vagrant-debian-12" - ], - "service.type": "system", - "user.group.id": "0", - "user.id": "0" - }, - { - "event.dataset": "system.syslog", - "event.kind": "event", - "event.module": "system", - "event.timezone": "-02:00", - "fileset.name": "syslog", - "host.hostname": "vagrant-debian-12", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 0, - "log.syslog.priority": 6, - "message": "Console: switching to colour frame buffer device 160x50", - "process.name": "", - "process.pid": "", - "related.hosts": [ - "vagrant-debian-12" - ], - "service.type": "system" - }, - { - "event.dataset": "system.syslog", - "event.kind": "event", - "event.module": "system", - "event.timezone": "-02:00", - "fileset.name": "syslog", - "host.hostname": "bookworm", - "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", - "input.type": "journald", - "log.syslog.facility.code": 0, - "log.syslog.priority": 6, - "message": "thermal_sys: Registered thermal governor 'power_allocator'", - "process.name": "", - "process.pid": "", - "related.hosts": [ - "bookworm" - ], - "service.type": "system" - } -] \ No newline at end of file diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled index 809b32de2ed..1302c6374da 100644 --- a/filebeat/modules.d/system.yml.disabled +++ b/filebeat/modules.d/system.yml.disabled @@ -10,16 +10,6 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - # Authorization logs auth: enabled: false @@ -27,20 +17,3 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # A list of tags to include in events. Including forwarded - # indicates that the events did not originate on this host and - # causes host.name to not be added to events. Include - # preserve_orginal_event causes the pipeline to retain the raw log - # in event.original. Defaults to []. - #var.tags: [] diff --git a/filebeat/tests/integration/systemlogs_linux_test.go b/filebeat/tests/integration/systemlogs_linux_test.go index 7c7526c60e8..88af84734af 100644 --- a/filebeat/tests/integration/systemlogs_linux_test.go +++ b/filebeat/tests/integration/systemlogs_linux_test.go @@ -31,7 +31,8 @@ import ( // TestSystemLogsCanUseJournald aims to ensure the system-logs input can // correctly choose and start a journald input when the globs defined in // var.paths do not resolve to any file. -func TestSystemLogsCanUseJournaldInput(t *testing.T) { +func TestSystemModuleCanUseJournaldInput(t *testing.T) { + t.Skip("The system module is not using the system-logs input at the moment") filebeat := integration.NewBeat( t, "filebeat", diff --git a/filebeat/tests/integration/systemlogs_other_test.go b/filebeat/tests/integration/systemlogs_other_test.go index 45235f92514..42fc61b426d 100644 --- a/filebeat/tests/integration/systemlogs_other_test.go +++ b/filebeat/tests/integration/systemlogs_other_test.go @@ -29,6 +29,7 @@ import ( ) func TestSystemLogsCanUseLogInput(t *testing.T) { + t.Skip("The system module is not using the system-logs input at the moment") filebeat := integration.NewBeat( t, "filebeat", diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index c00099c3667..5e636901565 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -21,18 +21,7 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # Input configuration (advanced). - # Any input configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. #input: @@ -44,23 +33,6 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Force using journald to collect system logs - #var.use_journald: true|false - - # Force using log files to collect system logs - #var.use_files: true|false - - # If use_journald and use_files are false, then - # Filebeat will autodetect whether use to journald - # to collect system logs. - - # A list of tags to include in events. Including 'forwarded' - # indicates that the events did not originate on this host and - # causes host.name to not be added to events. Include - # 'preserve_orginal_event' causes the pipeline to retain the raw log - # in event.original. Defaults to []. - #var.tags: [] - # Input configuration (advanced). Any input configuration option # can be added under this section. #input: