[Winlogbeat] Add missing query while reading .evtx file (#36173)

* Add missing query for evtx processing

* update pr num

* update changelog

* Add test

* fix CI

* add eventID as string

* update query

* fix expected in test

* fix golangci-lint

* Address PR comment

* Add nolint:prealloc directives

(cherry picked from commit 1fe462c)

# Conflicts:
#	winlogbeat/eventlog/wineventlog.go
#	winlogbeat/eventlog/wineventlog_experimental.go

CHANGELOG.next.asciidoc

@@ -23,6 +23,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Winlogbeat*
 
+- Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193]
+- Fix the ability to use filtering features (e.g. `ignore_older`, `event_id`, `provider`, `level`) while reading `.evtx` files. {issue}16826[16826] {pull}36173[36173]
+
 *Functionbeat*
 
 ==== Bugfixes

winlogbeat/eventlog/wineventlog.go

@@ -595,6 +595,277 @@ func newWinEventLog(options *common.Config) (EventLog, error) {
 	return l, nil
 }
 
+<<<<<<< HEAD
+=======
+// Name returns the name of the event log (i.e. Application, Security, etc.).
+func (l *winEventLog) Name() string {
+	return l.id
+}
+
+// Channel returns the event log's channel name.
+func (l *winEventLog) Channel() string {
+	return l.channelName
+}
+
+// IsFile returns true if the event log is an evtx file.
+func (l *winEventLog) IsFile() bool {
+	return l.file
+}
+
+func (l *winEventLog) Open(state checkpoint.EventLogState) error {
+	var bookmark win.EvtHandle
+	var err error
+	if len(state.Bookmark) > 0 {
+		bookmark, err = win.CreateBookmarkFromXML(state.Bookmark)
+	} else if state.RecordNumber > 0 && l.channelName != "" {
+		bookmark, err = win.CreateBookmarkFromRecordID(l.channelName, state.RecordNumber)
+	}
+	if err != nil {
+		l.metrics.logError(err)
+		return err
+	}
+	defer win.Close(bookmark)
+
+	if l.file {
+		return l.openFile(state, bookmark)
+	}
+	return l.openChannel(bookmark)
+}
+
+func (l *winEventLog) openFile(state checkpoint.EventLogState, bookmark win.EvtHandle) error {
+	path := l.channelName
+
+	h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection)
+	if err != nil {
+		l.metrics.logError(err)
+		return fmt.Errorf("failed to get handle to event log file %v: %w", path, err)
+	}
+
+	if bookmark > 0 {
+		debugf("%s Seeking to bookmark. timestamp=%v bookmark=%v",
+			l.logPrefix, state.Timestamp, state.Bookmark)
+
+		// This seeks to the last read event and strictly validates that the
+		// bookmarked record number exists.
+		if err = win.EvtSeek(h, 0, bookmark, win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil {
+			// Then we advance past the last read event to avoid sending that
+			// event again. This won't fail if we're at the end of the file.
+			if seekErr := win.EvtSeek(h, 1, bookmark, win.EvtSeekRelativeToBookmark); seekErr != nil {
+				err = fmt.Errorf("failed to seek past bookmarked position: %w", seekErr)
+			}
+		} else {
+			logp.Warn("%s Failed to seek to bookmarked location in %v (error: %v). "+
+				"Recovering by reading the log from the beginning. (Did the file "+
+				"change since it was last read?)", l.logPrefix, path, err)
+			l.metrics.logError(err)
+			if seekErr := win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst); seekErr != nil {
+				err = fmt.Errorf("failed to seek to beginning of log: %w", seekErr)
+			}
+		}
+
+		if err != nil {
+			l.metrics.logError(err)
+			return err
+		}
+	}
+
+	l.subscription = h
+	return nil
+}
+
+func (l *winEventLog) openChannel(bookmark win.EvtHandle) error {
+	// Using a pull subscription to receive events. See:
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull
+	signalEvent, err := windows.CreateEvent(nil, 0, 0, nil)
+	if err != nil {
+		l.metrics.logError(err)
+		return err
+	}
+	defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release.
+
+	var flags win.EvtSubscribeFlag
+	if bookmark > 0 {
+		// Use EvtSubscribeStrict to detect when the bookmark is missing and be able to
+		// subscribe again from the beginning.
+		flags = win.EvtSubscribeStartAfterBookmark | win.EvtSubscribeStrict
+	} else {
+		flags = win.EvtSubscribeStartAtOldestRecord
+	}
+
+	debugf("%s using subscription query=%s", l.logPrefix, l.query)
+	subscriptionHandle, err := win.Subscribe(
+		0, // Session - nil for localhost
+		signalEvent,
+		"",       // Channel - empty b/c channel is in the query
+		l.query,  // Query - nil means all events
+		bookmark, // Bookmark - for resuming from a specific event
+		flags)
+
+	switch {
+	case errors.Is(err, win.ERROR_NOT_FOUND), errors.Is(err, win.ERROR_EVT_QUERY_RESULT_STALE),
+		errors.Is(err, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION):
+		debugf("%s error subscribing (first chance): %v", l.logPrefix, err)
+		// The bookmarked event was not found, we retry the subscription from the start.
+		l.metrics.logError(err)
+		incrementMetric(readErrors, err)
+		subscriptionHandle, err = win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord)
+	}
+
+	if err != nil {
+		l.metrics.logError(err)
+		debugf("%s error subscribing (final): %v", l.logPrefix, err)
+		return err
+	}
+
+	l.subscription = subscriptionHandle
+	return nil
+}
+
+func (l *winEventLog) Read() ([]Record, error) {
+	handles, _, err := l.eventHandles(l.maxRead)
+	if err != nil || len(handles) == 0 {
+		return nil, err
+	}
+
+	//nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive.
+	var records []Record
+	defer func() {
+		l.metrics.log(records)
+		for _, h := range handles {
+			win.Close(h)
+		}
+	}()
+	detailf("%s EventHandles returned %d handles", l.logPrefix, len(handles))
+
+	for _, h := range handles {
+		l.outputBuf.Reset()
+		err := l.render(h, l.outputBuf)
+		var bufErr sys.InsufficientBufferError
+		if errors.As(err, &bufErr) {
+			detailf("%s Increasing render buffer size to %d", l.logPrefix,
+				bufErr.RequiredSize)
+			l.renderBuf = make([]byte, bufErr.RequiredSize)
+			l.outputBuf.Reset()
+			err = l.render(h, l.outputBuf)
+		}
+		l.metrics.logError(err)
+		if err != nil && l.outputBuf.Len() == 0 {
+			logp.Err("%s Dropping event with rendering error. %v", l.logPrefix, err)
+			l.metrics.logDropped(err)
+			incrementMetric(dropReasons, err)
+			continue
+		}
+
+		r := l.buildRecordFromXML(l.outputBuf.Bytes(), err)
+		r.Offset = checkpoint.EventLogState{
+			Name:         l.id,
+			RecordNumber: r.RecordID,
+			Timestamp:    r.TimeCreated.SystemTime,
+		}
+		if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil {
+			l.metrics.logError(err)
+			logp.Warn("%s failed creating bookmark: %v", l.logPrefix, err)
+		}
+		if r.Message == "" && l.message != nil {
+			r.Message, err = l.message(h)
+			if err != nil {
+				l.metrics.logError(err)
+				logp.Warn("%s error salvaging message (event id=%d qualifier=%d provider=%q created at %s will be included without a message): %v",
+					l.logPrefix, r.EventIdentifier.ID, r.EventIdentifier.Qualifiers, r.Provider.Name, r.TimeCreated.SystemTime, err)
+			}
+		}
+		records = append(records, r)
+		l.lastRead = r.Offset
+	}
+
+	debugf("%s Read() is returning %d records", l.logPrefix, len(records))
+	return records, nil
+}
+
+func (l *winEventLog) eventHandles(maxRead int) ([]win.EvtHandle, int, error) {
+	handles, err := win.EventHandles(l.subscription, maxRead)
+	switch err { //nolint:errorlint // This is an errno or nil.
+	case nil:
+		if l.maxRead > maxRead {
+			debugf("%s Recovered from RPC_S_INVALID_BOUND error (errno 1734) "+
+				"by decreasing batch_read_size to %v", l.logPrefix, maxRead)
+		}
+		return handles, maxRead, nil
+	case win.ERROR_NO_MORE_ITEMS:
+		detailf("%s No more events", l.logPrefix)
+		if l.config.NoMoreEvents == Stop {
+			return nil, maxRead, io.EOF
+		}
+		return nil, maxRead, nil
+	case win.RPC_S_INVALID_BOUND:
+		incrementMetric(readErrors, err)
+		l.metrics.logError(err)
+		if err := l.Close(); err != nil {
+			return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err)
+		}
+		if err := l.Open(l.lastRead); err != nil {
+			return nil, 0, fmt.Errorf("failed to recover from RPC_S_INVALID_BOUND: %w", err)
+		}
+		return l.eventHandles(maxRead / 2)
+	default:
+		l.metrics.logError(err)
+		incrementMetric(readErrors, err)
+		logp.Warn("%s EventHandles returned error %v", l.logPrefix, err)
+		return nil, 0, err
+	}
+}
+
+func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) Record {
+	includeXML := l.config.IncludeXML
+	e, err := winevent.UnmarshalXML(x)
+	if err != nil {
+		e.RenderErr = append(e.RenderErr, err.Error())
+		// Add raw XML to event.original when decoding fails
+		includeXML = true
+	}
+
+	err = winevent.PopulateAccount(&e.User)
+	if err != nil {
+		debugf("%s SID %s account lookup failed. %v", l.logPrefix,
+			e.User.Identifier, err)
+	}
+
+	if e.RenderErrorCode != 0 {
+		// Convert the render error code to an error message that can be
+		// included in the "error.message" field.
+		e.RenderErr = append(e.RenderErr, syscall.Errno(e.RenderErrorCode).Error())
+	} else if recoveredErr != nil {
+		e.RenderErr = append(e.RenderErr, recoveredErr.Error())
+	}
+
+	// Get basic string values for raw fields.
+	winevent.EnrichRawValuesWithNames(l.winMeta(e.Provider.Name), &e)
+	if e.Level == "" {
+		// Fallback on LevelRaw if the Level is not set in the RenderingInfo.
+		e.Level = win.EventLevel(e.LevelRaw).String()
+	}
+
+	if logp.IsDebug(detailSelector) {
+		detailf("%s XML=%s Event=%+v", l.logPrefix, x, e)
+	}
+
+	r := Record{
+		API:   winEventLogAPIName,
+		Event: e,
+	}
+
+	if l.file {
+		r.File = l.id
+	}
+
+	if includeXML {
+		r.XML = string(x)
+	}
+
+	return r
+}
+
+>>>>>>> 1fe462c68f ([Winlogbeat] Add missing query while reading .evtx file (#36173))
 func (l *winEventLog) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) {
 	bmHandle, err := win.CreateBookmarkFromEvent(evtHandle)
 	if err != nil {