From 1b438c1a4fcd71523ae4e23b32519bb90c7687f6 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Reddy Burri Date: Tue, 8 Aug 2023 13:01:25 +0530 Subject: [PATCH 1/3] [Winlogbeat] Add missing query while reading .evtx file (#36173) * Add missing query for evtx processing * update pr num * update changelog * Add test * fix CI * add eventID as string * update query * fix expected in test * fix golangci-lint * Address PR comment * Add nolint:prealloc directives (cherry picked from commit 1fe462c68f7d3c1c0b526acfec1aa74cf57e38ce) # Conflicts: # winlogbeat/eventlog/wineventlog.go # winlogbeat/eventlog/wineventlog_experimental.go --- CHANGELOG.next.asciidoc | 3 + winlogbeat/eventlog/wineventlog.go | 271 ++++++++++++++++++ .../eventlog/wineventlog_experimental.go | 220 ++++++++++++++ winlogbeat/eventlog/wineventlog_test.go | 29 ++ 4 files changed, 523 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2a0d7834096..08e79a2dd8d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -23,6 +23,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Winlogbeat* +- Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193] +- Fix the ability to use filtering features (e.g. `ignore_older`, `event_id`, `provider`, `level`) while reading `.evtx` files. {issue}16826[16826] {pull}36173[36173] + *Functionbeat* ==== Bugfixes diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 77b57603cc6..9677073da0e 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -595,6 +595,277 @@ func newWinEventLog(options *common.Config) (EventLog, error) { return l, nil } +<<<<<<< HEAD +======= +// Name returns the name of the event log (i.e. Application, Security, etc.). +func (l *winEventLog) Name() string { + return l.id +} + +// Channel returns the event log's channel name. +func (l *winEventLog) Channel() string { + return l.channelName +} + +// IsFile returns true if the event log is an evtx file. +func (l *winEventLog) IsFile() bool { + return l.file +} + +func (l *winEventLog) Open(state checkpoint.EventLogState) error { + var bookmark win.EvtHandle + var err error + if len(state.Bookmark) > 0 { + bookmark, err = win.CreateBookmarkFromXML(state.Bookmark) + } else if state.RecordNumber > 0 && l.channelName != "" { + bookmark, err = win.CreateBookmarkFromRecordID(l.channelName, state.RecordNumber) + } + if err != nil { + l.metrics.logError(err) + return err + } + defer win.Close(bookmark) + + if l.file { + return l.openFile(state, bookmark) + } + return l.openChannel(bookmark) +} + +func (l *winEventLog) openFile(state checkpoint.EventLogState, bookmark win.EvtHandle) error { + path := l.channelName + + h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) + if err != nil { + l.metrics.logError(err) + return fmt.Errorf("failed to get handle to event log file %v: %w", path, err) + } + + if bookmark > 0 { + debugf("%s Seeking to bookmark. timestamp=%v bookmark=%v", + l.logPrefix, state.Timestamp, state.Bookmark) + + // This seeks to the last read event and strictly validates that the + // bookmarked record number exists. + if err = win.EvtSeek(h, 0, bookmark, win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { + // Then we advance past the last read event to avoid sending that + // event again. This won't fail if we're at the end of the file. + if seekErr := win.EvtSeek(h, 1, bookmark, win.EvtSeekRelativeToBookmark); seekErr != nil { + err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr) + } + } else { + logp.Warn("%s Failed to seek to bookmarked location in %v (error: %v). "+ + "Recovering by reading the log from the beginning. (Did the file "+ + "change since it was last read?)", l.logPrefix, path, err) + l.metrics.logError(err) + if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil { + err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr) + } + } + + if err != nil { + l.metrics.logError(err) + return err + } + } + + l.subscription = h + return nil +} + +func (l *winEventLog) openChannel(bookmark win.EvtHandle) error { + // Using a pull subscription to receive events. See: + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull + signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + l.metrics.logError(err) + return err + } + defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. + + var flags win.EvtSubscribeFlag + if bookmark > 0 { + // Use EvtSubscribeStrict to detect when the bookmark is missing and be able to + // subscribe again from the beginning. + flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict + } else { + flags = win.EvtSubscribeStartAtOldestRecord + } + + debugf("%s using subscription query=%s", l.logPrefix, l.query) + subscriptionHandle, err := win.Subscribe( + 0, // Session - nil for localhost + signalEvent, + "", // Channel - empty b/c channel is in the query + l.query, // Query - nil means all events + bookmark, // Bookmark - for resuming from a specific event + flags) + + switch { + case errors.Is(err, win.ERROR_NOT_FOUND), errors.Is(err, win.ERROR_EVT_QUERY_RESULT_STALE), + errors.Is(err, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION): + debugf("%s error subscribing (first chance): %v", l.logPrefix, err) + // The bookmarked event was not found, we retry the subscription from the start. + l.metrics.logError(err) + incrementMetric(readErrors, err) + subscriptionHandle, err = win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) + } + + if err != nil { + l.metrics.logError(err) + debugf("%s error subscribing (final): %v", l.logPrefix, err) + return err + } + + l.subscription = subscriptionHandle + return nil +} + +func (l *winEventLog) Read() ([]Record, error) { + handles, _, err := l.eventHandles(l.maxRead) + if err != nil || len(handles) == 0 { + return nil, err + } + + //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. + var records []Record + defer func() { + l.metrics.log(records) + for _, h := range handles { + win.Close(h) + } + }() + detailf("%s EventHandles returned %d handles", l.logPrefix, len(handles)) + + for _, h := range handles { + l.outputBuf.Reset() + err := l.render(h, l.outputBuf) + var bufErr sys.InsufficientBufferError + if errors.As(err, &bufErr) { + detailf("%s Increasing render buffer size to %d", l.logPrefix, + bufErr.RequiredSize) + l.renderBuf = make([]byte, bufErr.RequiredSize) + l.outputBuf.Reset() + err = l.render(h, l.outputBuf) + } + l.metrics.logError(err) + if err != nil && l.outputBuf.Len() == 0 { + logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err) + l.metrics.logDropped(err) + incrementMetric(dropReasons, err) + continue + } + + r := l.buildRecordFromXML(l.outputBuf.Bytes(), err) + r.Offset = checkpoint.EventLogState{ + Name: l.id, + RecordNumber: r.RecordID, + Timestamp: r.TimeCreated.SystemTime, + } + if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { + l.metrics.logError(err) + logp.Warn("%s failed creating bookmark: %v", l.logPrefix, err) + } + if r.Message == "" && l.message != nil { + r.Message, err = l.message(h) + if err != nil { + l.metrics.logError(err) + logp.Warn("%s error salvaging message (event id=%d qualifier=%d provider=%q created at %s will be included without a message): %v", + l.logPrefix, r.EventIdentifier.ID, r.EventIdentifier.Qualifiers, r.Provider.Name, r.TimeCreated.SystemTime, err) + } + } + records = append(records, r) + l.lastRead = r.Offset + } + + debugf("%s Read() is returning %d records", l.logPrefix, len(records)) + return records, nil +} + +func (l *winEventLog) eventHandles(maxRead int) ([]win.EvtHandle, int, error) { + handles, err := win.EventHandles(l.subscription, maxRead) + switch err { //nolint:errorlint // This is an errno or nil. + case nil: + if l.maxRead > maxRead { + debugf("%s Recovered from RPC_S_INVALID_BOUND error (errno 1734) "+ + "by decreasing batch_read_size to %v", l.logPrefix, maxRead) + } + return handles, maxRead, nil + case win.ERROR_NO_MORE_ITEMS: + detailf("%s No more events", l.logPrefix) + if l.config.NoMoreEvents == Stop { + return nil, maxRead, io.EOF + } + return nil, maxRead, nil + case win.RPC_S_INVALID_BOUND: + incrementMetric(readErrors, err) + l.metrics.logError(err) + if err := l.Close(); err != nil { + return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err) + } + if err := l.Open(l.lastRead); err != nil { + return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err) + } + return l.eventHandles(maxRead / 2) + default: + l.metrics.logError(err) + incrementMetric(readErrors, err) + logp.Warn("%s EventHandles returned error %v", l.logPrefix, err) + return nil, 0, err + } +} + +func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) Record { + includeXML := l.config.IncludeXML + e, err := winevent.UnmarshalXML(x) + if err != nil { + e.RenderErr = append(e.RenderErr, err.Error()) + // Add raw XML to event.original when decoding fails + includeXML = true + } + + err = winevent.PopulateAccount(&e.User) + if err != nil { + debugf("%s SID %s account lookup failed. %v", l.logPrefix, + e.User.Identifier, err) + } + + if e.RenderErrorCode != 0 { + // Convert the render error code to an error message that can be + // included in the "error.message" field. + e.RenderErr = append(e.RenderErr, syscall.Errno(e.RenderErrorCode).Error()) + } else if recoveredErr != nil { + e.RenderErr = append(e.RenderErr, recoveredErr.Error()) + } + + // Get basic string values for raw fields. + winevent.EnrichRawValuesWithNames(l.winMeta(e.Provider.Name), &e) + if e.Level == "" { + // Fallback on LevelRaw if the Level is not set in the RenderingInfo. + e.Level = win.EventLevel(e.LevelRaw).String() + } + + if logp.IsDebug(detailSelector) { + detailf("%s XML=%s Event=%+v", l.logPrefix, x, e) + } + + r := Record{ + API: winEventLogAPIName, + Event: e, + } + + if l.file { + r.File = l.id + } + + if includeXML { + r.XML = string(x) + } + + return r +} + +>>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173)) func (l *winEventLog) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { bmHandle, err := win.CreateBookmarkFromEvent(evtHandle) if err != nil { diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go index dd80096a4b6..e858e998e03 100644 --- a/winlogbeat/eventlog/wineventlog_experimental.go +++ b/winlogbeat/eventlog/wineventlog_experimental.go @@ -337,3 +337,223 @@ func newWinEventLogExp(options *common.Config) (EventLog, error) { return l, nil } +<<<<<<< HEAD +======= + +// Name returns the name of the event log (i.e. Application, Security, etc.). +func (l *winEventLogExp) Name() string { + return l.id +} + +// Channel returns the event log's channel name. +func (l *winEventLogExp) Channel() string { + return l.channelName +} + +// IsFile returns true if the event log is an evtx file. +func (l *winEventLogExp) IsFile() bool { + return l.file +} + +func (l *winEventLogExp) Open(state checkpoint.EventLogState) error { + l.lastRead = state + + var err error + l.iterator, err = win.NewEventIterator( + win.WithSubscriptionFactory(func() (handle win.EvtHandle, err error) { + return l.open(l.lastRead) + }), + win.WithBatchSize(l.maxRead)) + return err +} + +func (l *winEventLogExp) open(state checkpoint.EventLogState) (win.EvtHandle, error) { + var bookmark win.Bookmark + if len(state.Bookmark) > 0 { + var err error + bookmark, err = win.NewBookmarkFromXML(state.Bookmark) + if err != nil { + return win.NilHandle, err + } + defer bookmark.Close() + } + + if l.file { + return l.openFile(state, bookmark) + } + return l.openChannel(bookmark) +} + +func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { + path := l.channelName + + h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) + if err != nil { + return win.NilHandle, fmt.Errorf("failed to get handle to event log file %v: %w", path, err) + } + + if bookmark > 0 { + l.log.Debugf("Seeking to bookmark. timestamp=%v bookmark=%v", + state.Timestamp, state.Bookmark) + + // This seeks to the last read event and strictly validates that the + // bookmarked record number exists. + if err = win.EvtSeek(h, 0, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { + // Then we advance past the last read event to avoid sending that + // event again. This won't fail if we're at the end of the file. + if seekErr := win.EvtSeek(h, 1, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark); seekErr != nil { + err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr) + } + } else { + l.log.Warnf("s Failed to seek to bookmarked location in %v (error: %v). "+ + "Recovering by reading the log from the beginning. (Did the file "+ + "change since it was last read?)", path, err) + if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil { + err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr) + } + } + + if err != nil { + return win.NilHandle, err + } + } + + return h, err +} + +func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) { + // Using a pull subscription to receive events. See: + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull + signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + return win.NilHandle, err + } + defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. + + var flags win.EvtSubscribeFlag + if bookmark > 0 { + // Use EvtSubscribeStrict to detect when the bookmark is missing and be able to + // subscribe again from the beginning. + flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict + } else { + flags = win.EvtSubscribeStartAtOldestRecord + } + + l.log.Debugw("Using subscription query.", "winlog.query", l.query) + h, err := win.Subscribe( + 0, // Session - nil for localhost + signalEvent, + "", // Channel - empty b/c channel is in the query + l.query, // Query - nil means all events + win.EvtHandle(bookmark), // Bookmark - for resuming from a specific event + flags) + + switch err { //nolint:errorlint // This is an errno or nil. + case nil: + return h, nil + case win.ERROR_NOT_FOUND, win.ERROR_EVT_QUERY_RESULT_STALE, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION: + // The bookmarked event was not found, we retry the subscription from the start. + incrementMetric(readErrors, err) + return win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) + default: + return 0, err + } +} + +func (l *winEventLogExp) Read() ([]Record, error) { + //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. + var records []Record + defer func() { + l.metrics.log(records) + }() + + for h, ok := l.iterator.Next(); ok; h, ok = l.iterator.Next() { + record, err := l.processHandle(h) + if err != nil { + l.metrics.logError(err) + l.log.Warnw("Dropping event due to rendering error.", "error", err) + l.metrics.logDropped(err) + incrementMetric(dropReasons, err) + continue + } + records = append(records, *record) + + // It has read the maximum requested number of events. + if len(records) >= l.maxRead { + return records, nil + } + } + + // An error occurred while retrieving more events. + if err := l.iterator.Err(); err != nil { + l.metrics.logError(err) + return records, err + } + + // Reader is configured to stop when there are no more events. + if Stop == l.config.NoMoreEvents { + return records, io.EOF + } + + return records, nil +} + +func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) { + defer h.Close() + + // NOTE: Render can return an error and a partial event. + evt, err := l.renderer.Render(h) + if evt == nil { + return nil, err + } + if err != nil { + evt.RenderErr = append(evt.RenderErr, err.Error()) + } + + //nolint:godox // Bad linter! Keep to have a record of feature disparity between non-experimental vs experimental. + // TODO: Need to add XML when configured. + + r := &Record{ + API: winEventLogExpAPIName, + Event: *evt, + } + + if l.file { + r.File = l.id + } + + r.Offset = checkpoint.EventLogState{ + Name: l.id, + RecordNumber: r.RecordID, + Timestamp: r.TimeCreated.SystemTime, + } + if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { + l.metrics.logError(err) + l.log.Warnw("Failed creating bookmark.", "error", err) + } + l.lastRead = r.Offset + return r, nil +} + +func (l *winEventLogExp) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { + bookmark, err := win.NewBookmarkFromEvent(evtHandle) + if err != nil { + return "", fmt.Errorf("failed to create new bookmark from event handle: %w", err) + } + defer bookmark.Close() + + return bookmark.XML() +} + +func (l *winEventLogExp) Close() error { + l.log.Debug("Closing event log reader handles.") + l.metrics.close() + if l.iterator == nil { + return l.renderer.Close() + } + return multierr.Combine( + l.iterator.Close(), + l.renderer.Close(), + ) +} +>>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173)) diff --git a/winlogbeat/eventlog/wineventlog_test.go b/winlogbeat/eventlog/wineventlog_test.go index da67c61b3e1..694810b6b68 100644 --- a/winlogbeat/eventlog/wineventlog_test.go +++ b/winlogbeat/eventlog/wineventlog_test.go @@ -276,6 +276,7 @@ func testWindowsEventLog(t *testing.T, api string) { assert.Equal(t, totalEvents, eventCount) }) + // Test reading .evtx file without any query filters t.Run("evtx_file", func(t *testing.T) { path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") if err != nil { @@ -301,6 +302,34 @@ func testWindowsEventLog(t *testing.T, api string) { assert.Len(t, records, 32) }) + + // Test reading .evtx file with event_id filter + t.Run("evtx_file_with_query", func(t *testing.T) { + path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") + if err != nil { + t.Fatal(err) + } + + log := openLog(t, map[string]interface{}{ + "name": path, + "no_more_events": "stop", + "event_id": "3, 5", + }) + defer log.Close() + + records, err := log.Read() + + // This implementation returns the EOF on the next call. + if err == nil && api == winEventLogAPIName { + _, err = log.Read() + } + + if assert.Error(t, err, "no_more_events=stop requires io.EOF to be returned") { + assert.Equal(t, io.EOF, err) + } + + assert.Len(t, records, 21) + }) } // ---- Utility Functions ----- From 47ba97ed5699c5a254dcae3501c60eae5bf25c91 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 10 Aug 2023 15:14:20 +0530 Subject: [PATCH 2/3] add query to evtx file open --- winlogbeat/eventlog/wineventlog.go | 273 +----------------- .../eventlog/wineventlog_experimental.go | 224 +------------- 2 files changed, 3 insertions(+), 494 deletions(-) diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 9677073da0e..2c64e60a2c5 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -279,7 +279,7 @@ func (l *winEventLog) openChannel(bookmark win.EvtHandle) error { func (l *winEventLog) openFile(state checkpoint.EventLogState, bookmark win.EvtHandle) error { path := l.channelName - h, err := win.EvtQuery(0, path, "", win.EvtQueryFilePath|win.EvtQueryForwardDirection) + h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) if err != nil { return fmt.Errorf("failed to get handle to event log file %v: %w", path, err) } @@ -595,277 +595,6 @@ func newWinEventLog(options *common.Config) (EventLog, error) { return l, nil } -<<<<<<< HEAD -======= -// Name returns the name of the event log (i.e. Application, Security, etc.). -func (l *winEventLog) Name() string { - return l.id -} - -// Channel returns the event log's channel name. -func (l *winEventLog) Channel() string { - return l.channelName -} - -// IsFile returns true if the event log is an evtx file. -func (l *winEventLog) IsFile() bool { - return l.file -} - -func (l *winEventLog) Open(state checkpoint.EventLogState) error { - var bookmark win.EvtHandle - var err error - if len(state.Bookmark) > 0 { - bookmark, err = win.CreateBookmarkFromXML(state.Bookmark) - } else if state.RecordNumber > 0 && l.channelName != "" { - bookmark, err = win.CreateBookmarkFromRecordID(l.channelName, state.RecordNumber) - } - if err != nil { - l.metrics.logError(err) - return err - } - defer win.Close(bookmark) - - if l.file { - return l.openFile(state, bookmark) - } - return l.openChannel(bookmark) -} - -func (l *winEventLog) openFile(state checkpoint.EventLogState, bookmark win.EvtHandle) error { - path := l.channelName - - h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) - if err != nil { - l.metrics.logError(err) - return fmt.Errorf("failed to get handle to event log file %v: %w", path, err) - } - - if bookmark > 0 { - debugf("%s Seeking to bookmark. timestamp=%v bookmark=%v", - l.logPrefix, state.Timestamp, state.Bookmark) - - // This seeks to the last read event and strictly validates that the - // bookmarked record number exists. - if err = win.EvtSeek(h, 0, bookmark, win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { - // Then we advance past the last read event to avoid sending that - // event again. This won't fail if we're at the end of the file. - if seekErr := win.EvtSeek(h, 1, bookmark, win.EvtSeekRelativeToBookmark); seekErr != nil { - err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr) - } - } else { - logp.Warn("%s Failed to seek to bookmarked location in %v (error: %v). "+ - "Recovering by reading the log from the beginning. (Did the file "+ - "change since it was last read?)", l.logPrefix, path, err) - l.metrics.logError(err) - if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil { - err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr) - } - } - - if err != nil { - l.metrics.logError(err) - return err - } - } - - l.subscription = h - return nil -} - -func (l *winEventLog) openChannel(bookmark win.EvtHandle) error { - // Using a pull subscription to receive events. See: - // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull - signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) - if err != nil { - l.metrics.logError(err) - return err - } - defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. - - var flags win.EvtSubscribeFlag - if bookmark > 0 { - // Use EvtSubscribeStrict to detect when the bookmark is missing and be able to - // subscribe again from the beginning. - flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict - } else { - flags = win.EvtSubscribeStartAtOldestRecord - } - - debugf("%s using subscription query=%s", l.logPrefix, l.query) - subscriptionHandle, err := win.Subscribe( - 0, // Session - nil for localhost - signalEvent, - "", // Channel - empty b/c channel is in the query - l.query, // Query - nil means all events - bookmark, // Bookmark - for resuming from a specific event - flags) - - switch { - case errors.Is(err, win.ERROR_NOT_FOUND), errors.Is(err, win.ERROR_EVT_QUERY_RESULT_STALE), - errors.Is(err, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION): - debugf("%s error subscribing (first chance): %v", l.logPrefix, err) - // The bookmarked event was not found, we retry the subscription from the start. - l.metrics.logError(err) - incrementMetric(readErrors, err) - subscriptionHandle, err = win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) - } - - if err != nil { - l.metrics.logError(err) - debugf("%s error subscribing (final): %v", l.logPrefix, err) - return err - } - - l.subscription = subscriptionHandle - return nil -} - -func (l *winEventLog) Read() ([]Record, error) { - handles, _, err := l.eventHandles(l.maxRead) - if err != nil || len(handles) == 0 { - return nil, err - } - - //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. - var records []Record - defer func() { - l.metrics.log(records) - for _, h := range handles { - win.Close(h) - } - }() - detailf("%s EventHandles returned %d handles", l.logPrefix, len(handles)) - - for _, h := range handles { - l.outputBuf.Reset() - err := l.render(h, l.outputBuf) - var bufErr sys.InsufficientBufferError - if errors.As(err, &bufErr) { - detailf("%s Increasing render buffer size to %d", l.logPrefix, - bufErr.RequiredSize) - l.renderBuf = make([]byte, bufErr.RequiredSize) - l.outputBuf.Reset() - err = l.render(h, l.outputBuf) - } - l.metrics.logError(err) - if err != nil && l.outputBuf.Len() == 0 { - logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err) - l.metrics.logDropped(err) - incrementMetric(dropReasons, err) - continue - } - - r := l.buildRecordFromXML(l.outputBuf.Bytes(), err) - r.Offset = checkpoint.EventLogState{ - Name: l.id, - RecordNumber: r.RecordID, - Timestamp: r.TimeCreated.SystemTime, - } - if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { - l.metrics.logError(err) - logp.Warn("%s failed creating bookmark: %v", l.logPrefix, err) - } - if r.Message == "" && l.message != nil { - r.Message, err = l.message(h) - if err != nil { - l.metrics.logError(err) - logp.Warn("%s error salvaging message (event id=%d qualifier=%d provider=%q created at %s will be included without a message): %v", - l.logPrefix, r.EventIdentifier.ID, r.EventIdentifier.Qualifiers, r.Provider.Name, r.TimeCreated.SystemTime, err) - } - } - records = append(records, r) - l.lastRead = r.Offset - } - - debugf("%s Read() is returning %d records", l.logPrefix, len(records)) - return records, nil -} - -func (l *winEventLog) eventHandles(maxRead int) ([]win.EvtHandle, int, error) { - handles, err := win.EventHandles(l.subscription, maxRead) - switch err { //nolint:errorlint // This is an errno or nil. - case nil: - if l.maxRead > maxRead { - debugf("%s Recovered from RPC_S_INVALID_BOUND error (errno 1734) "+ - "by decreasing batch_read_size to %v", l.logPrefix, maxRead) - } - return handles, maxRead, nil - case win.ERROR_NO_MORE_ITEMS: - detailf("%s No more events", l.logPrefix) - if l.config.NoMoreEvents == Stop { - return nil, maxRead, io.EOF - } - return nil, maxRead, nil - case win.RPC_S_INVALID_BOUND: - incrementMetric(readErrors, err) - l.metrics.logError(err) - if err := l.Close(); err != nil { - return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err) - } - if err := l.Open(l.lastRead); err != nil { - return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err) - } - return l.eventHandles(maxRead / 2) - default: - l.metrics.logError(err) - incrementMetric(readErrors, err) - logp.Warn("%s EventHandles returned error %v", l.logPrefix, err) - return nil, 0, err - } -} - -func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) Record { - includeXML := l.config.IncludeXML - e, err := winevent.UnmarshalXML(x) - if err != nil { - e.RenderErr = append(e.RenderErr, err.Error()) - // Add raw XML to event.original when decoding fails - includeXML = true - } - - err = winevent.PopulateAccount(&e.User) - if err != nil { - debugf("%s SID %s account lookup failed. %v", l.logPrefix, - e.User.Identifier, err) - } - - if e.RenderErrorCode != 0 { - // Convert the render error code to an error message that can be - // included in the "error.message" field. - e.RenderErr = append(e.RenderErr, syscall.Errno(e.RenderErrorCode).Error()) - } else if recoveredErr != nil { - e.RenderErr = append(e.RenderErr, recoveredErr.Error()) - } - - // Get basic string values for raw fields. - winevent.EnrichRawValuesWithNames(l.winMeta(e.Provider.Name), &e) - if e.Level == "" { - // Fallback on LevelRaw if the Level is not set in the RenderingInfo. - e.Level = win.EventLevel(e.LevelRaw).String() - } - - if logp.IsDebug(detailSelector) { - detailf("%s XML=%s Event=%+v", l.logPrefix, x, e) - } - - r := Record{ - API: winEventLogAPIName, - Event: e, - } - - if l.file { - r.File = l.id - } - - if includeXML { - r.XML = string(x) - } - - return r -} - ->>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173)) func (l *winEventLog) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { bmHandle, err := win.CreateBookmarkFromEvent(evtHandle) if err != nil { diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go index e858e998e03..4c5c0ee4883 100644 --- a/winlogbeat/eventlog/wineventlog_experimental.go +++ b/winlogbeat/eventlog/wineventlog_experimental.go @@ -152,7 +152,7 @@ func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, erro func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { path := l.channelName - h, err := win.EvtQuery(0, path, "", win.EvtQueryFilePath|win.EvtQueryForwardDirection) + h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) if err != nil { return win.NilHandle, fmt.Errorf("failed to get handle to event log file %v: %w", path, err) } @@ -187,7 +187,7 @@ func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.B } func (l *winEventLogExp) Read() ([]Record, error) { - var records []Record + var records []Record //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. for h, ok := l.iterator.Next(); ok; h, ok = l.iterator.Next() { record, err := l.processHandle(h) @@ -337,223 +337,3 @@ func newWinEventLogExp(options *common.Config) (EventLog, error) { return l, nil } -<<<<<<< HEAD -======= - -// Name returns the name of the event log (i.e. Application, Security, etc.). -func (l *winEventLogExp) Name() string { - return l.id -} - -// Channel returns the event log's channel name. -func (l *winEventLogExp) Channel() string { - return l.channelName -} - -// IsFile returns true if the event log is an evtx file. -func (l *winEventLogExp) IsFile() bool { - return l.file -} - -func (l *winEventLogExp) Open(state checkpoint.EventLogState) error { - l.lastRead = state - - var err error - l.iterator, err = win.NewEventIterator( - win.WithSubscriptionFactory(func() (handle win.EvtHandle, err error) { - return l.open(l.lastRead) - }), - win.WithBatchSize(l.maxRead)) - return err -} - -func (l *winEventLogExp) open(state checkpoint.EventLogState) (win.EvtHandle, error) { - var bookmark win.Bookmark - if len(state.Bookmark) > 0 { - var err error - bookmark, err = win.NewBookmarkFromXML(state.Bookmark) - if err != nil { - return win.NilHandle, err - } - defer bookmark.Close() - } - - if l.file { - return l.openFile(state, bookmark) - } - return l.openChannel(bookmark) -} - -func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { - path := l.channelName - - h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection) - if err != nil { - return win.NilHandle, fmt.Errorf("failed to get handle to event log file %v: %w", path, err) - } - - if bookmark > 0 { - l.log.Debugf("Seeking to bookmark. timestamp=%v bookmark=%v", - state.Timestamp, state.Bookmark) - - // This seeks to the last read event and strictly validates that the - // bookmarked record number exists. - if err = win.EvtSeek(h, 0, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { - // Then we advance past the last read event to avoid sending that - // event again. This won't fail if we're at the end of the file. - if seekErr := win.EvtSeek(h, 1, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark); seekErr != nil { - err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr) - } - } else { - l.log.Warnf("s Failed to seek to bookmarked location in %v (error: %v). "+ - "Recovering by reading the log from the beginning. (Did the file "+ - "change since it was last read?)", path, err) - if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil { - err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr) - } - } - - if err != nil { - return win.NilHandle, err - } - } - - return h, err -} - -func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) { - // Using a pull subscription to receive events. See: - // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull - signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) - if err != nil { - return win.NilHandle, err - } - defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. - - var flags win.EvtSubscribeFlag - if bookmark > 0 { - // Use EvtSubscribeStrict to detect when the bookmark is missing and be able to - // subscribe again from the beginning. - flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict - } else { - flags = win.EvtSubscribeStartAtOldestRecord - } - - l.log.Debugw("Using subscription query.", "winlog.query", l.query) - h, err := win.Subscribe( - 0, // Session - nil for localhost - signalEvent, - "", // Channel - empty b/c channel is in the query - l.query, // Query - nil means all events - win.EvtHandle(bookmark), // Bookmark - for resuming from a specific event - flags) - - switch err { //nolint:errorlint // This is an errno or nil. - case nil: - return h, nil - case win.ERROR_NOT_FOUND, win.ERROR_EVT_QUERY_RESULT_STALE, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION: - // The bookmarked event was not found, we retry the subscription from the start. - incrementMetric(readErrors, err) - return win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) - default: - return 0, err - } -} - -func (l *winEventLogExp) Read() ([]Record, error) { - //nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive. - var records []Record - defer func() { - l.metrics.log(records) - }() - - for h, ok := l.iterator.Next(); ok; h, ok = l.iterator.Next() { - record, err := l.processHandle(h) - if err != nil { - l.metrics.logError(err) - l.log.Warnw("Dropping event due to rendering error.", "error", err) - l.metrics.logDropped(err) - incrementMetric(dropReasons, err) - continue - } - records = append(records, *record) - - // It has read the maximum requested number of events. - if len(records) >= l.maxRead { - return records, nil - } - } - - // An error occurred while retrieving more events. - if err := l.iterator.Err(); err != nil { - l.metrics.logError(err) - return records, err - } - - // Reader is configured to stop when there are no more events. - if Stop == l.config.NoMoreEvents { - return records, io.EOF - } - - return records, nil -} - -func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) { - defer h.Close() - - // NOTE: Render can return an error and a partial event. - evt, err := l.renderer.Render(h) - if evt == nil { - return nil, err - } - if err != nil { - evt.RenderErr = append(evt.RenderErr, err.Error()) - } - - //nolint:godox // Bad linter! Keep to have a record of feature disparity between non-experimental vs experimental. - // TODO: Need to add XML when configured. - - r := &Record{ - API: winEventLogExpAPIName, - Event: *evt, - } - - if l.file { - r.File = l.id - } - - r.Offset = checkpoint.EventLogState{ - Name: l.id, - RecordNumber: r.RecordID, - Timestamp: r.TimeCreated.SystemTime, - } - if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { - l.metrics.logError(err) - l.log.Warnw("Failed creating bookmark.", "error", err) - } - l.lastRead = r.Offset - return r, nil -} - -func (l *winEventLogExp) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { - bookmark, err := win.NewBookmarkFromEvent(evtHandle) - if err != nil { - return "", fmt.Errorf("failed to create new bookmark from event handle: %w", err) - } - defer bookmark.Close() - - return bookmark.XML() -} - -func (l *winEventLogExp) Close() error { - l.log.Debug("Closing event log reader handles.") - l.metrics.close() - if l.iterator == nil { - return l.renderer.Close() - } - return multierr.Combine( - l.iterator.Close(), - l.renderer.Close(), - ) -} ->>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173)) From 489173686b0b80c97699b9befc981ea14f84afcb Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 10 Aug 2023 15:15:49 +0530 Subject: [PATCH 3/3] remove unnecessary change log --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 08e79a2dd8d..737e6426217 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -23,7 +23,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Winlogbeat* -- Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193] - Fix the ability to use filtering features (e.g. `ignore_older`, `event_id`, `provider`, `level`) while reading `.evtx` files. {issue}16826[16826] {pull}36173[36173] *Functionbeat*