diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2a0d7834096..08e79a2dd8d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -23,6 +23,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Winlogbeat* +- Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193] +- Fix the ability to use filtering features (e.g. `ignore_older`, `event_id`, `provider`, `level`) while reading `.evtx` files. {issue}16826[16826] {pull}36173[36173] + *Functionbeat* ==== Bugfixes diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 77b57603cc6..9677073da0e 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -595,6 +595,277 @@ func newWinEventLog(options *common.Config) (EventLog, error) { return l, nil } +<<<<<<< HEAD +======= +// Name returns the name of the event log (i.e. Application, Security, etc.). +func (l *winEventLog) Name() string { + return l.id +} + +// Channel returns the event log's channel name. +func (l *winEventLog) Channel() string { + return l.channelName +} + +// IsFile returns true if the event log is an evtx file. +func (l *winEventLog) IsFile() bool { + return l.file +} + +func (l *winEventLog) Open(state checkpoint.EventLogState) error { + var bookmark win.EvtHandle + var err error + if len(state.Bookmark) > 0 { + bookmark, err = win.CreateBookmarkFromXML(state.Bookmark) + } else if state.RecordNumber > 0 && l.channelName != "" { + bookmark, err = win.CreateBookmarkFromRecordID(l.channelName, state.RecordNumber) + } + if err != nil { + l.metrics.logError(err) + return err + } + defer win.Close(bookmark) + + if l.file { + return l.openFile(state, bookmark) + } + return l.openChannel(bookmark) +} + +func (l *winEventLog) openFile(state checkpoint.EventLogState, bookmark win.EvtHandle) error { + path := l.channelName + + h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) + if err != nil { + l.metrics.logError(err) + return fmt.Errorf("failed to get handle to event log file %v: %w", path, err) + } + + if bookmark > 0 { + debugf("%s Seeking to bookmark. timestamp=%v bookmark=%v", + l.logPrefix, state.Timestamp, state.Bookmark) + + // This seeks to the last read event and strictly validates that the + // bookmarked record number exists. + if err = win.EvtSeek(h, 0, bookmark, win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { + // Then we advance past the last read event to avoid sending that + // event again. This won't fail if we're at the end of the file. + if seekErr := win.EvtSeek(h, 1, bookmark, win.EvtSeekRelativeToBookmark); seekErr != nil { + err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr) + } + } else { + logp.Warn("%s Failed to seek to bookmarked location in %v (error: %v). "+ + "Recovering by reading the log from the beginning. (Did the file "+ + "change since it was last read?)", l.logPrefix, path, err) + l.metrics.logError(err) + if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil { + err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr) + } + } + + if err != nil { + l.metrics.logError(err) + return err + } + } + + l.subscription = h + return nil +} + +func (l *winEventLog) openChannel(bookmark win.EvtHandle) error { + // Using a pull subscription to receive events. See: + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull + signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + l.metrics.logError(err) + return err + } + defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. + + var flags win.EvtSubscribeFlag + if bookmark > 0 { + // Use EvtSubscribeStrict to detect when the bookmark is missing and be able to + // subscribe again from the beginning. + flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict + } else { + flags = win.EvtSubscribeStartAtOldestRecord + } + + debugf("%s using subscription query=%s", l.logPrefix, l.query) + subscriptionHandle, err := win.Subscribe( + 0, // Session - nil for localhost + signalEvent, + "", // Channel - empty b/c channel is in the query + l.query, // Query - nil means all events + bookmark, // Bookmark - for resuming from a specific event + flags) + + switch { + case errors.Is(err, win.ERROR_NOT_FOUND), errors.Is(err, win.ERROR_EVT_QUERY_RESULT_STALE), + errors.Is(err, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION): + debugf("%s error subscribing (first chance): %v", l.logPrefix, err) + // The bookmarked event was not found, we retry the subscription from the start. + l.metrics.logError(err) + incrementMetric(readErrors, err) + subscriptionHandle, err = win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) + } + + if err != nil { + l.metrics.logError(err) + debugf("%s error subscribing (final): %v", l.logPrefix, err) + return err + } + + l.subscription = subscriptionHandle + return nil +} + +func (l *winEventLog) Read() ([]Record, error) { + handles, _, err := l.eventHandles(l.maxRead) + if err != nil || len(handles) == 0 { + return nil, err + } + + //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. + var records []Record + defer func() { + l.metrics.log(records) + for _, h := range handles { + win.Close(h) + } + }() + detailf("%s EventHandles returned %d handles", l.logPrefix, len(handles)) + + for _, h := range handles { + l.outputBuf.Reset() + err := l.render(h, l.outputBuf) + var bufErr sys.InsufficientBufferError + if errors.As(err, &bufErr) { + detailf("%s Increasing render buffer size to %d", l.logPrefix, + bufErr.RequiredSize) + l.renderBuf = make([]byte, bufErr.RequiredSize) + l.outputBuf.Reset() + err = l.render(h, l.outputBuf) + } + l.metrics.logError(err) + if err != nil && l.outputBuf.Len() == 0 { + logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err) + l.metrics.logDropped(err) + incrementMetric(dropReasons, err) + continue + } + + r := l.buildRecordFromXML(l.outputBuf.Bytes(), err) + r.Offset = checkpoint.EventLogState{ + Name: l.id, + RecordNumber: r.RecordID, + Timestamp: r.TimeCreated.SystemTime, + } + if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { + l.metrics.logError(err) + logp.Warn("%s failed creating bookmark: %v", l.logPrefix, err) + } + if r.Message == "" && l.message != nil { + r.Message, err = l.message(h) + if err != nil { + l.metrics.logError(err) + logp.Warn("%s error salvaging message (event id=%d qualifier=%d provider=%q created at %s will be included without a message): %v", + l.logPrefix, r.EventIdentifier.ID, r.EventIdentifier.Qualifiers, r.Provider.Name, r.TimeCreated.SystemTime, err) + } + } + records = append(records, r) + l.lastRead = r.Offset + } + + debugf("%s Read() is returning %d records", l.logPrefix, len(records)) + return records, nil +} + +func (l *winEventLog) eventHandles(maxRead int) ([]win.EvtHandle, int, error) { + handles, err := win.EventHandles(l.subscription, maxRead) + switch err { //nolint:errorlint // This is an errno or nil. + case nil: + if l.maxRead > maxRead { + debugf("%s Recovered from RPC_S_INVALID_BOUND error (errno 1734) "+ + "by decreasing batch_read_size to %v", l.logPrefix, maxRead) + } + return handles, maxRead, nil + case win.ERROR_NO_MORE_ITEMS: + detailf("%s No more events", l.logPrefix) + if l.config.NoMoreEvents == Stop { + return nil, maxRead, io.EOF + } + return nil, maxRead, nil + case win.RPC_S_INVALID_BOUND: + incrementMetric(readErrors, err) + l.metrics.logError(err) + if err := l.Close(); err != nil { + return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err) + } + if err := l.Open(l.lastRead); err != nil { + return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err) + } + return l.eventHandles(maxRead / 2) + default: + l.metrics.logError(err) + incrementMetric(readErrors, err) + logp.Warn("%s EventHandles returned error %v", l.logPrefix, err) + return nil, 0, err + } +} + +func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) Record { + includeXML := l.config.IncludeXML + e, err := winevent.UnmarshalXML(x) + if err != nil { + e.RenderErr = append(e.RenderErr, err.Error()) + // Add raw XML to event.original when decoding fails + includeXML = true + } + + err = winevent.PopulateAccount(&e.User) + if err != nil { + debugf("%s SID %s account lookup failed. %v", l.logPrefix, + e.User.Identifier, err) + } + + if e.RenderErrorCode != 0 { + // Convert the render error code to an error message that can be + // included in the "error.message" field. + e.RenderErr = append(e.RenderErr, syscall.Errno(e.RenderErrorCode).Error()) + } else if recoveredErr != nil { + e.RenderErr = append(e.RenderErr, recoveredErr.Error()) + } + + // Get basic string values for raw fields. + winevent.EnrichRawValuesWithNames(l.winMeta(e.Provider.Name), &e) + if e.Level == "" { + // Fallback on LevelRaw if the Level is not set in the RenderingInfo. + e.Level = win.EventLevel(e.LevelRaw).String() + } + + if logp.IsDebug(detailSelector) { + detailf("%s XML=%s Event=%+v", l.logPrefix, x, e) + } + + r := Record{ + API: winEventLogAPIName, + Event: e, + } + + if l.file { + r.File = l.id + } + + if includeXML { + r.XML = string(x) + } + + return r +} + +>>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173)) func (l *winEventLog) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { bmHandle, err := win.CreateBookmarkFromEvent(evtHandle) if err != nil { diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go index dd80096a4b6..e858e998e03 100644 --- a/winlogbeat/eventlog/wineventlog_experimental.go +++ b/winlogbeat/eventlog/wineventlog_experimental.go @@ -337,3 +337,223 @@ func newWinEventLogExp(options *common.Config) (EventLog, error) { return l, nil } +<<<<<<< HEAD +======= + +// Name returns the name of the event log (i.e. Application, Security, etc.). +func (l *winEventLogExp) Name() string { + return l.id +} + +// Channel returns the event log's channel name. +func (l *winEventLogExp) Channel() string { + return l.channelName +} + +// IsFile returns true if the event log is an evtx file. +func (l *winEventLogExp) IsFile() bool { + return l.file +} + +func (l *winEventLogExp) Open(state checkpoint.EventLogState) error { + l.lastRead = state + + var err error + l.iterator, err = win.NewEventIterator( + win.WithSubscriptionFactory(func() (handle win.EvtHandle, err error) { + return l.open(l.lastRead) + }), + win.WithBatchSize(l.maxRead)) + return err +} + +func (l *winEventLogExp) open(state checkpoint.EventLogState) (win.EvtHandle, error) { + var bookmark win.Bookmark + if len(state.Bookmark) > 0 { + var err error + bookmark, err = win.NewBookmarkFromXML(state.Bookmark) + if err != nil { + return win.NilHandle, err + } + defer bookmark.Close() + } + + if l.file { + return l.openFile(state, bookmark) + } + return l.openChannel(bookmark) +} + +func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { + path := l.channelName + + h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) + if err != nil { + return win.NilHandle, fmt.Errorf("failed to get handle to event log file %v: %w", path, err) + } + + if bookmark > 0 { + l.log.Debugf("Seeking to bookmark. timestamp=%v bookmark=%v", + state.Timestamp, state.Bookmark) + + // This seeks to the last read event and strictly validates that the + // bookmarked record number exists. + if err = win.EvtSeek(h, 0, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { + // Then we advance past the last read event to avoid sending that + // event again. This won't fail if we're at the end of the file. + if seekErr := win.EvtSeek(h, 1, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark); seekErr != nil { + err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr) + } + } else { + l.log.Warnf("s Failed to seek to bookmarked location in %v (error: %v). "+ + "Recovering by reading the log from the beginning. (Did the file "+ + "change since it was last read?)", path, err) + if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil { + err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr) + } + } + + if err != nil { + return win.NilHandle, err + } + } + + return h, err +} + +func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) { + // Using a pull subscription to receive events. See: + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull + signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + return win.NilHandle, err + } + defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. + + var flags win.EvtSubscribeFlag + if bookmark > 0 { + // Use EvtSubscribeStrict to detect when the bookmark is missing and be able to + // subscribe again from the beginning. + flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict + } else { + flags = win.EvtSubscribeStartAtOldestRecord + } + + l.log.Debugw("Using subscription query.", "winlog.query", l.query) + h, err := win.Subscribe( + 0, // Session - nil for localhost + signalEvent, + "", // Channel - empty b/c channel is in the query + l.query, // Query - nil means all events + win.EvtHandle(bookmark), // Bookmark - for resuming from a specific event + flags) + + switch err { //nolint:errorlint // This is an errno or nil. + case nil: + return h, nil + case win.ERROR_NOT_FOUND, win.ERROR_EVT_QUERY_RESULT_STALE, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION: + // The bookmarked event was not found, we retry the subscription from the start. + incrementMetric(readErrors, err) + return win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) + default: + return 0, err + } +} + +func (l *winEventLogExp) Read() ([]Record, error) { + //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. + var records []Record + defer func() { + l.metrics.log(records) + }() + + for h, ok := l.iterator.Next(); ok; h, ok = l.iterator.Next() { + record, err := l.processHandle(h) + if err != nil { + l.metrics.logError(err) + l.log.Warnw("Dropping event due to rendering error.", "error", err) + l.metrics.logDropped(err) + incrementMetric(dropReasons, err) + continue + } + records = append(records, *record) + + // It has read the maximum requested number of events. + if len(records) >= l.maxRead { + return records, nil + } + } + + // An error occurred while retrieving more events. + if err := l.iterator.Err(); err != nil { + l.metrics.logError(err) + return records, err + } + + // Reader is configured to stop when there are no more events. + if Stop == l.config.NoMoreEvents { + return records, io.EOF + } + + return records, nil +} + +func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) { + defer h.Close() + + // NOTE: Render can return an error and a partial event. + evt, err := l.renderer.Render(h) + if evt == nil { + return nil, err + } + if err != nil { + evt.RenderErr = append(evt.RenderErr, err.Error()) + } + + //nolint:godox // Bad linter! Keep to have a record of feature disparity between non-experimental vs experimental. + // TODO: Need to add XML when configured. + + r := &Record{ + API: winEventLogExpAPIName, + Event: *evt, + } + + if l.file { + r.File = l.id + } + + r.Offset = checkpoint.EventLogState{ + Name: l.id, + RecordNumber: r.RecordID, + Timestamp: r.TimeCreated.SystemTime, + } + if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { + l.metrics.logError(err) + l.log.Warnw("Failed creating bookmark.", "error", err) + } + l.lastRead = r.Offset + return r, nil +} + +func (l *winEventLogExp) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { + bookmark, err := win.NewBookmarkFromEvent(evtHandle) + if err != nil { + return "", fmt.Errorf("failed to create new bookmark from event handle: %w", err) + } + defer bookmark.Close() + + return bookmark.XML() +} + +func (l *winEventLogExp) Close() error { + l.log.Debug("Closing event log reader handles.") + l.metrics.close() + if l.iterator == nil { + return l.renderer.Close() + } + return multierr.Combine( + l.iterator.Close(), + l.renderer.Close(), + ) +} +>>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173)) diff --git a/winlogbeat/eventlog/wineventlog_test.go b/winlogbeat/eventlog/wineventlog_test.go index da67c61b3e1..694810b6b68 100644 --- a/winlogbeat/eventlog/wineventlog_test.go +++ b/winlogbeat/eventlog/wineventlog_test.go @@ -276,6 +276,7 @@ func testWindowsEventLog(t *testing.T, api string) { assert.Equal(t, totalEvents, eventCount) }) + // Test reading .evtx file without any query filters t.Run("evtx_file", func(t *testing.T) { path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") if err != nil { @@ -301,6 +302,34 @@ func testWindowsEventLog(t *testing.T, api string) { assert.Len(t, records, 32) }) + + // Test reading .evtx file with event_id filter + t.Run("evtx_file_with_query", func(t *testing.T) { + path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") + if err != nil { + t.Fatal(err) + } + + log := openLog(t, map[string]interface{}{ + "name": path, + "no_more_events": "stop", + "event_id": "3, 5", + }) + defer log.Close() + + records, err := log.Read() + + // This implementation returns the EOF on the next call. + if err == nil && api == winEventLogAPIName { + _, err = log.Read() + } + + if assert.Error(t, err, "no_more_events=stop requires io.EOF to be returned") { + assert.Equal(t, io.EOF, err) + } + + assert.Len(t, records, 21) + }) } // ---- Utility Functions -----