Merge branch 'main' into platform-specific-models

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
@@ -107,6 +108,7 @@ private static Operator operator(String data, int topCount) {
             ClusterSettings.createBuiltInClusterSettings()
         );
         return new TopNOperator(
+            BlockFactory.getNonBreakingInstance(),
             breakerService.getBreaker(CircuitBreaker.REQUEST),
             topCount,
             elementTypes,

docs/changelog/99947.yaml

@@ -0,0 +1,5 @@
+pr: 99947
+summary: GET `_data_stream` displays both ILM and DSL information
+area: Data streams
+type: feature
+issues: []

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -573,15 +573,21 @@ stream's oldest backing index.
       "indices": [
         {
           "index_name": ".ds-my-data-stream-2099.03.07-000001", <1>
-          "index_uuid": "Gpdiyq8sRuK9WuthvAdFbw"
+          "index_uuid": "Gpdiyq8sRuK9WuthvAdFbw",
+          "prefer_ilm": true,
+          "managed_by": "Unmanaged"
         },
         {
           "index_name": ".ds-my-data-stream-2099.03.08-000002",
-          "index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"
+          "index_uuid": "_eEfRrFHS9OyhqWntkgHAQ",
+          "prefer_ilm": true,
+          "managed_by": "Unmanaged"
         }
       ],
       "generation": 2,
       "status": "GREEN",
+      "next_generation_managed_by": "Unmanaged",
+      "prefer_ilm": true,
       "template": "my-data-stream-template",
       "hidden": false,
       "system": false,

docs/reference/data-streams/downsampling-manual.asciidoc

@@ -358,11 +358,15 @@ This returns:
       "indices": [
         {
           "index_name": ".ds-my-data-stream-2023.07.26-000001", <1>
-          "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg"
+          "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg",
+          "prefer_ilm": true,
+          "managed_by": "Data stream lifecycle"
         }
       ],
       "generation": 1,
       "status": "GREEN",
+      "next_generation_managed_by": "Data stream lifecycle",
+      "prefer_ilm": true,
       "template": "my-data-stream-template",
       "hidden": false,
       "system": false,

docs/reference/indices/get-data-stream.asciidoc

@@ -225,7 +225,7 @@ cluster can not write into this data stream or change its mappings.
 
 `lifecycle`::
 (object)
-Functionality in preview:[]. Contains the configuration for the data stream lifecycle management of this data stream.
+Contains the configuration for the data stream lifecycle management of this data stream.
 +
 .Properties of `lifecycle`
 [%collapsible%open]
@@ -265,18 +265,26 @@ The API returns the following response:
       "indices": [
         {
           "index_name": ".ds-my-data-stream-2099.03.07-000001",
-          "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"
+          "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
+          "prefer_ilm": true,
+          "ilm_policy": "my-lifecycle-policy",
+          "managed_by": "Index Lifecycle Management"
         },
         {
           "index_name": ".ds-my-data-stream-2099.03.08-000002",
-          "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"
+          "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
+          "prefer_ilm": true,
+          "ilm_policy": "my-lifecycle-policy",
+          "managed_by": "Index Lifecycle Management"
         }
       ],
       "generation": 2,
       "_meta": {
         "my-meta-field": "foo"
       },
       "status": "GREEN",
+      "next_generation_managed_by": "Index Lifecycle Management",
+      "prefer_ilm": true,
       "template": "my-index-template",
       "ilm_policy": "my-lifecycle-policy",
       "hidden": false,
@@ -292,14 +300,19 @@ The API returns the following response:
       "indices": [
         {
           "index_name": ".ds-my-data-stream-two-2099.03.08-000001",
-          "index_uuid": "3liBu2SYS5axasRt6fUIpA"
+          "index_uuid": "3liBu2SYS5axasRt6fUIpA",
+          "prefer_ilm": true,
+          "ilm_policy": "my-lifecycle-policy",
+          "managed_by": "Index Lifecycle Management"
         }
       ],
       "generation": 1,
       "_meta": {
         "my-meta-field": "foo"
       },
       "status": "YELLOW",
+      "next_generation_managed_by": "Index Lifecycle Management",
+      "prefer_ilm": true,
       "template": "my-index-template",
       "ilm_policy": "my-lifecycle-policy",
       "hidden": false,

libs/core/src/main/java/org/elasticsearch/core/Releasables.java

@@ -89,7 +89,7 @@ private static void close(boolean success, Releasable... releasables) {
      *  // the resources will be released when reaching here
      *  </pre>
      */
-    public static Releasable wrap(final Iterable<Releasable> releasables) {
+    public static Releasable wrap(final Iterable<? extends Releasable> releasables) {
         return new Releasable() {
             @Override
             public void close() {

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -1320,11 +1320,17 @@ public void testGetDataStream() throws Exception {
         ).actionGet();
         assertThat(response.getDataStreams().size(), is(1));
         DataStreamInfo metricsFooDataStream = response.getDataStreams().get(0);
-        assertThat(metricsFooDataStream.getDataStream().getName(), is("metrics-foo"));
+        DataStream dataStream = metricsFooDataStream.getDataStream();
+        assertThat(dataStream.getName(), is("metrics-foo"));
         assertThat(metricsFooDataStream.getDataStreamStatus(), is(ClusterHealthStatus.YELLOW));
         assertThat(metricsFooDataStream.getIndexTemplate(), is("template_for_foo"));
         assertThat(metricsFooDataStream.getIlmPolicy(), is(nullValue()));
-        assertThat(metricsFooDataStream.getDataStream().getLifecycle(), is(lifecycle));
+        assertThat(dataStream.getLifecycle(), is(lifecycle));
+        assertThat(metricsFooDataStream.templatePreferIlmValue(), is(true));
+        GetDataStreamAction.Response.IndexProperties indexProperties = metricsFooDataStream.getIndexSettingsValues()
+            .get(dataStream.getWriteIndex());
+        assertThat(indexProperties.ilmPolicyName(), is(nullValue()));
+        assertThat(indexProperties.preferIlm(), is(true));
     }
 
     private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping, Map<?, ?> expectedMapping) {

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java

@@ -11,6 +11,8 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
+import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.IndexProperties;
+import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.ManagedBy;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.cluster.ClusterState;
@@ -21,6 +23,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -39,9 +42,12 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING;
+
 public class GetDataStreamsTransportAction extends TransportMasterNodeReadAction<
     GetDataStreamAction.Request,
     GetDataStreamAction.Response> {
@@ -95,6 +101,7 @@ static GetDataStreamAction.Response innerOperation(
         List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = new ArrayList<>(dataStreams.size());
         for (DataStream dataStream : dataStreams) {
             final String indexTemplate;
+            boolean indexTemplatePreferIlmValue = true;
             String ilmPolicyName = null;
             if (dataStream.isSystem()) {
                 SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName());
@@ -104,13 +111,15 @@ static GetDataStreamAction.Response innerOperation(
                         dataStreamDescriptor.getComposableIndexTemplate(),
                         dataStreamDescriptor.getComponentTemplates()
                     );
-                    ilmPolicyName = settings.get("index.lifecycle.name");
+                    ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
+                    indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
                 }
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
                     Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
-                    ilmPolicyName = settings.get("index.lifecycle.name");
+                    ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
+                    indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
                 } else {
                     LOGGER.warn(
                         "couldn't find any matching template for data stream [{}]. has it been restored (and possibly renamed)"
@@ -125,18 +134,35 @@ static GetDataStreamAction.Response innerOperation(
                 dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new)
             );
 
+            Map<Index, IndexProperties> backingIndicesSettingsValues = new HashMap<>();
+            Metadata metadata = state.getMetadata();
+            for (Index index : dataStream.getIndices()) {
+                IndexMetadata indexMetadata = metadata.index(index);
+                Boolean preferIlm = PREFER_ILM_SETTING.get(indexMetadata.getSettings());
+                assert preferIlm != null : "must use the default prefer ilm setting value, if nothing else";
+                ManagedBy managedBy;
+                if (metadata.isIndexManagedByILM(indexMetadata)) {
+                    managedBy = ManagedBy.ILM;
+                } else if (dataStream.isIndexManagedByDataStreamLifecycle(index, metadata::index)) {
+                    managedBy = ManagedBy.LIFECYCLE;
+                } else {
+                    managedBy = ManagedBy.UNMANAGED;
+                }
+                backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy));
+            }
+
             GetDataStreamAction.Response.TimeSeries timeSeries = null;
             if (dataStream.getIndexMode() == IndexMode.TIME_SERIES) {
                 List<Tuple<Instant, Instant>> ranges = new ArrayList<>();
                 Tuple<Instant, Instant> current = null;
                 String previousIndexName = null;
                 for (Index index : dataStream.getIndices()) {
-                    IndexMetadata metadata = state.getMetadata().index(index);
-                    if (metadata.getIndexMode() != IndexMode.TIME_SERIES) {
+                    IndexMetadata indexMetadata = metadata.index(index);
+                    if (indexMetadata.getIndexMode() != IndexMode.TIME_SERIES) {
                         continue;
                     }
-                    Instant start = metadata.getTimeSeriesStart();
-                    Instant end = metadata.getTimeSeriesEnd();
+                    Instant start = indexMetadata.getTimeSeriesStart();
+                    Instant end = indexMetadata.getTimeSeriesEnd();
                     if (current == null) {
                         current = new Tuple<>(start, end);
                     } else if (current.v2().compareTo(start) == 0) {
@@ -175,7 +201,9 @@ static GetDataStreamAction.Response innerOperation(
                     streamHealth.getStatus(),
                     indexTemplate,
                     ilmPolicyName,
-                    timeSeries
+                    timeSeries,
+                    backingIndicesSettingsValues,
+                    indexTemplatePreferIlmValue
                 )
             );
         }