[microsoft-exchange-online-message-trace] - Added support for sliding window attributes and updated default interval values.

packages/microsoft_exchange_online_message_trace/_dev/build/docs/README.md

@@ -45,9 +45,13 @@ In order to continue using the Microsoft Exchange Online Message Trace you will
 - For configuring `Local Domains` you can check your [Microsoft Admin Exchange Center](https://admin.exchange.microsoft.com/) for the domains
 available in your organization. They are usually under the sections [Accepted Domains](https://admin.exchange.microsoft.com/#/accepteddomains) and [Remote Domains](https://admin.exchange.microsoft.com/#/remotedomains).
 
-- The default `Polling Interval` and `Initial Interval` values are configured to `1h`, you can however change these to your required values. The look-back 
+- The default `Interval` is configured to `1h` and `Initial Interval` to `48h`, you can however change these to your required values. The look-back 
   value of `Initial Interval` should not exceed `200 hours` as this might cause unexpected errors with the API.
 
+- The default `Minimum Age` is configured to `1h`, you can however change these to your required values. The `Minimum Age` was introduced to allow a sliding 
+  window to exist in combination with the `Initial Interval`. If you do not require a sliding window you can set this to `0s` which will cause the `Minimum Age` to 
+  always default to the `current time (now)`.
+
 - The default `Additional Look-back Time` value is configured for `1h`. 
   This is intended to capture events that may not have been initially present due to eventual consistency.
   This value does not need to exceed [`24h`](https://learn.microsoft.com/en-us/previous-versions/office/developer/o365-enterprise-developers/jj984335(v=office.15)#data-granularity-persistence-and-availability).

packages/microsoft_exchange_online_message_trace/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.25.1"
+  changes:
+    - description: Added the 'end_interval' input parameter to support sliding window logic with updates to some default values.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1111
 - version: "1.25.0"
   changes:
     - description: Do not remove `event.original` in main ingest pipeline.

packages/microsoft_exchange_online_message_trace/data_stream/log/agent/stream/httpjson.yml.hbs

@@ -29,23 +29,26 @@ request.transforms:
   - set:
       # Warning: Modification of the `$filter` param format can break the `last_execution_datetime` cursor
       target: url.params.$filter
-      value: StartDate eq datetime'[[(formatDate ((parseDate .cursor.last_execution_datetime "RFC3339").Add (parseDuration "-{{additional_look_back}}")) "RFC3339")]]' and EndDate eq datetime'[[(formatDate (now) "RFC3339")]]'
-      default: StartDate eq datetime'[[(formatDate (now (parseDuration "-{{initial_interval}}")) "RFC3339")]]' and EndDate eq datetime'[[(formatDate (now) "RFC3339")]]'
+      value: StartDate eq datetime'[[(formatDate ((parseDate .cursor.last_execution_datetime "RFC3339").Add (parseDuration "-{{additional_look_back}}")) "RFC3339")]]' and EndDate eq datetime'[[(formatDate (now (parseDuration "-{{min_age}}")) "RFC3339")]]'
+      default: StartDate eq datetime'[[(formatDate (now (parseDuration "-{{initial_interval}}")) "RFC3339")]]' and EndDate eq datetime'[[(formatDate (now (parseDuration "-{{min_age}}")) "RFC3339")]]'
   - set:
       target: url.params.$top
       value: {{batch_size}}
   - set:
       target: url.params.$skiptoken
       value: 0
 fields_under_root: true
-{{#if local_domains}}
 fields:
   _conf:
+  {{#if local_domains}}
     local_domains:
     {{#each local_domains as |local_domain i|}}
       - {{local_domain}}
     {{/each}}
-{{/if}}
+  {{/if}}
+  {{#if drop_status}}
+    drop_status: {{drop_status}}
+  {{/if}}
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event

packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -17,6 +17,8 @@ processors:
   # Drop "null" event
   - drop:
       if: ctx.microsoft?.online_message_trace?.value instanceof List && ctx.microsoft.online_message_trace.value.length == 0 
+  - drop:
+      if: ctx.microsoft?.online_message_trace?.Status?.equalsIgnoreCase(ctx._conf?.Status) == true
   - set:
       field: event.outcome
       value: success

packages/microsoft_exchange_online_message_trace/docs/README.md

@@ -45,9 +45,13 @@ In order to continue using the Microsoft Exchange Online Message Trace you will
 - For configuring `Local Domains` you can check your [Microsoft Admin Exchange Center](https://admin.exchange.microsoft.com/) for the domains
 available in your organization. They are usually under the sections [Accepted Domains](https://admin.exchange.microsoft.com/#/accepteddomains) and [Remote Domains](https://admin.exchange.microsoft.com/#/remotedomains).
 
-- The default `Polling Interval` and `Initial Interval` values are configured to `1h`, you can however change these to your required values. The look-back 
+- The default `Interval` is configured to `1h` and `Initial Interval` to `48h`, you can however change these to your required values. The look-back 
   value of `Initial Interval` should not exceed `200 hours` as this might cause unexpected errors with the API.
 
+- The default `Minimum Age` is configured to `1h`, you can however change these to your required values. The `Minimum Age` was introduced to allow a sliding 
+  window to exist in combination with the `Initial Interval`. If you do not require a sliding window you can set this to `0s` which will cause the `Minimum Age` to 
+  always default to the `current time (now)`.
+
 - The default `Additional Look-back Time` value is configured for `1h`. 
   This is intended to capture events that may not have been initially present due to eventual consistency.
   This value does not need to exceed [`24h`](https://learn.microsoft.com/en-us/previous-versions/office/developer/o365-enterprise-developers/jj984335(v=office.15)#data-granularity-persistence-and-availability).

packages/microsoft_exchange_online_message_trace/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: microsoft_exchange_online_message_trace
 title: "Microsoft Exchange Online Message Trace"
-version: "1.25.0"
+version: "1.25.1"
 description: "Microsoft Exchange Online Message Trace Integration"
 type: integration
 categories:
@@ -123,10 +123,27 @@ policy_templates:
             type: text
             title: Initial Interval
             description: How far back to pull the initial log from Exchange Online. Supported units for this parameter are h/m/s.
+            default: 48h
+            multi: false
+            required: true
+            show_user: true
+          - name: min_age
+            type: text
+            title: Minimum Age
+            description: |
+              Logs will not be requested until they are at least this old. This value should be always lesser in value than the initial_interval.
+              Supported units for this parameter are h/m/s.
             default: 1h
             multi: false
             required: true
             show_user: true
+          - name: drop_status
+            type: text
+            title: Drop Logs With Status
+            description: Logs having the status string defined here will be dropped in the ingest pipeline.
+            multi: false
+            required: false
+            show_user: false
           - name: additional_look_back
             type: text
             title: Additional Look-back Time