Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…ensureUniqueModelIds
maxhniebergall · Dec 27, 2023 · f1b6192 · f1b6192
2 parents dfa97fb + f8b9490
commit f1b6192
Show file tree

Hide file tree

Showing 239 changed files with 3,028 additions and 1,886 deletions.
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java
@@ -76,6 +76,7 @@ public class BlockBenchmark {
     public static final String[] RELEVANT_TYPE_BLOCK_COMBINATIONS = {
         "boolean/array",
         "boolean/array-multivalue-null",
+        "boolean/big-array",
         "boolean/big-array-multivalue-null",
         "boolean/vector",
         "boolean/vector-big-array",
@@ -86,18 +87,21 @@ public class BlockBenchmark {
         "BytesRef/vector-const",
         "double/array",
         "double/array-multivalue-null",
+        "double/big-array",
         "double/big-array-multivalue-null",
         "double/vector",
         "double/vector-big-array",
         "double/vector-const",
         "int/array",
         "int/array-multivalue-null",
+        "int/big-array",
         "int/big-array-multivalue-null",
         "int/vector",
         "int/vector-big-array",
         "int/vector-const",
         "long/array",
         "long/array-multivalue-null",
+        "long/big-array",
         "long/big-array-multivalue-null",
         "long/vector",
         "long/vector-big-array",
@@ -177,6 +181,23 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                                 Block.MvOrdering.UNORDERED
                             );
                         }
+                        case "big-array" -> {
+                            BitArray valuesBigArray = new BitArray(totalPositions, BigArrays.NON_RECYCLING_INSTANCE);
+                            for (int i = 0; i < values.length; i++) {
+                                if (values[i]) {
+                                    valuesBigArray.set(i);
+                                }
+                            }
+
+                            blocks[blockIndex] = new BooleanBigArrayBlock(
+                                valuesBigArray,
+                                totalPositions,
+                                null,
+                                null,
+                                Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
+                                blockFactory
+                            );
+                        }
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
@@ -315,6 +336,21 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                                 Block.MvOrdering.UNORDERED
                             );
                         }
+                        case "big-array" -> {
+                            DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
+                            for (int i = 0; i < values.length; i++) {
+                                valuesBigArray.set(i, values[i]);
+                            }
+
+                            blocks[blockIndex] = new DoubleBigArrayBlock(
+                                valuesBigArray,
+                                totalPositions,
+                                null,
+                                null,
+                                Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
+                                blockFactory
+                            );
+                        }
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
@@ -392,6 +428,21 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                                 Block.MvOrdering.UNORDERED
                             );
                         }
+                        case "big-array" -> {
+                            IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
+                            for (int i = 0; i < values.length; i++) {
+                                valuesBigArray.set(i, values[i]);
+                            }
+
+                            blocks[blockIndex] = new IntBigArrayBlock(
+                                valuesBigArray,
+                                totalPositions,
+                                null,
+                                null,
+                                Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
+                                blockFactory
+                            );
+                        }
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
@@ -469,6 +520,21 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                                 Block.MvOrdering.UNORDERED
                             );
                         }
+                        case "big-array" -> {
+                            LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
+                            for (int i = 0; i < values.length; i++) {
+                                valuesBigArray.set(i, values[i]);
+                            }
+
+                            blocks[blockIndex] = new LongBigArrayBlock(
+                                valuesBigArray,
+                                totalPositions,
+                                null,
+                                null,
+                                Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
+                                blockFactory
+                            );
+                        }
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
@@ -715,6 +781,7 @@ private static boolean isRandom(String accessType) {
         {
             "boolean/array",
             "boolean/array-multivalue-null",
+            "boolean/big-array",
             "boolean/big-array-multivalue-null",
             "boolean/vector",
             "boolean/vector-big-array",
@@ -725,18 +792,21 @@ private static boolean isRandom(String accessType) {
             "BytesRef/vector-const",
             "double/array",
             "double/array-multivalue-null",
+            "double/big-array",
             "double/big-array-multivalue-null",
             "double/vector",
             "double/vector-big-array",
             "double/vector-const",
             "int/array",
             "int/array-multivalue-null",
+            "int/big-array",
             "int/big-array-multivalue-null",
             "int/vector",
             "int/vector-big-array",
             "int/vector-const",
             "long/array",
             "long/array-multivalue-null",
+            "long/big-array",
             "long/big-array-multivalue-null",
             "long/vector",
             "long/vector-big-array",

diff --git a/...c/main/java/org/elasticsearch/gradle/internal/packer/CacheTestFixtureResourcesPlugin.java b/...c/main/java/org/elasticsearch/gradle/internal/packer/CacheTestFixtureResourcesPlugin.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.gradle.internal.ResolveAllDependencies;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
@@ -26,9 +27,12 @@ public void apply(Project project) {
         var cacheTestFixturesConfiguration = project.getConfigurations().create(CACHE_TEST_FIXTURES);
         cacheTestFixturesConfiguration.defaultDependencies(deps -> {
             DependencyHandler dependencyHandler = project.getDependencies();
-            deps.add(dependencyHandler.create("org.reflections:reflections:" + VersionProperties.getVersions().get("reflections")));
-            deps.add(dependencyHandler.create("org.javassist:javassist:" + VersionProperties.getVersions().get("javassist")));
+            Dependency reflections = dependencyHandler.create(
+                "org.reflections:reflections:" + VersionProperties.getVersions().get("reflections")
+            );
+            deps.add(reflections);
         });
+
         project.getPlugins().withType(JavaPlugin.class, javaPlugin -> {
             var cacheTestFixtures = project.getTasks().register(CACHE_TEST_FIXTURES, CacheCacheableTestFixtures.class, (t) -> {
                 var testSourceSet = project.getExtensions()

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -48,8 +48,7 @@ ductTape          = 1.0.8
 commonsCompress   = 1.24.0
 
 # packer caching build logic
-reflections       = 0.9.12
-javassist         = 3.28.0-GA
+reflections       = 0.10.2
 
 # benchmark dependencies
 jmh               = 1.26

diff --git a/docs/changelog/103135.yaml b/docs/changelog/103135.yaml
diff --git a/docs/changelog/103591.yaml b/docs/changelog/103591.yaml
@@ -0,0 +1,6 @@
+pr: 103591
+summary: Wait for the model results on graceful shutdown
+area: Machine Learning
+type: bug
+issues:
+ - 103414
diff --git a/docs/changelog/103615.yaml b/docs/changelog/103615.yaml
@@ -0,0 +1,5 @@
+pr: 103615
+summary: Fix downsample api by returning a failure in case one or more downsample persistent tasks failed
+area: Downsampling
+type: bug
+issues: []
diff --git a/docs/changelog/103628.yaml b/docs/changelog/103628.yaml
@@ -0,0 +1,5 @@
+pr: 103628
+summary: Add ES|QL async delete API
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/103633.yaml b/docs/changelog/103633.yaml
@@ -0,0 +1,5 @@
+pr: 103633
+summary: Update s3 latency metric to use micros
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/103646.yaml b/docs/changelog/103646.yaml
@@ -0,0 +1,5 @@
+pr: 103646
+summary: Add index mapping parameter for `counted_keyword`
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/103669.yaml b/docs/changelog/103669.yaml
@@ -0,0 +1,5 @@
+pr: 103669
+summary: Validate inference model ids
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/103670.yaml b/docs/changelog/103670.yaml
@@ -0,0 +1,5 @@
+pr: 103670
+summary: "ESQL: Improve local folding of aggregates"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/103710.yaml b/docs/changelog/103710.yaml
@@ -0,0 +1,5 @@
+pr: 103710
+summary: List hidden shard stores by default
+area: Store
+type: enhancement
+issues: []
diff --git a/docs/reference/esql/esql-async-query-api.asciidoc b/docs/reference/esql/esql-async-query-api.asciidoc
@@ -80,3 +80,13 @@ finished, and the results are returned.
 }
 ----
 // TEST[skip: no access to search ID - may return response values]
+
+Use the <<delete-async-eqsl-query-api,delete async ES|QL query API>> to
+delete an async search before the `keep_alive` period ends. If the query
+is still running, {es} cancels it.
+
+[source,console]
+----
+DELETE /_query/async/delete/FmdMX2pIang3UWhLRU5QS0lqdlppYncaMUpYQ05oSkpTc3kwZ21EdC1tbFJXQToxOTI=
+----
+// TEST[skip: no access to search ID]
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.telemetry.tracing.SpanId;
+import org.elasticsearch.telemetry.tracing.Traceable;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -61,7 +61,7 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
     private static final Logger logger = LogManager.getLogger(APMTracer.class);
 
     /** Holds in-flight span information. */
-    private final Map<SpanId, Context> spans = ConcurrentCollections.newConcurrentMap();
+    private final Map<String, Context> spans = ConcurrentCollections.newConcurrentMap();
 
     private volatile boolean enabled;
     private volatile APMServices services;
@@ -160,8 +160,9 @@ private void destroyApmServices() {
     }
 
     @Override
-    public void startTrace(ThreadContext threadContext, SpanId spanId, String spanName, @Nullable Map<String, Object> attributes) {
+    public void startTrace(ThreadContext threadContext, Traceable traceable, String spanName, @Nullable Map<String, Object> attributes) {
         assert threadContext != null;
+        String spanId = traceable.getSpanId();
         assert spanId != null;
         assert spanName != null;
 
@@ -276,12 +277,12 @@ private Context getParentContext(ThreadContext threadContext) {
      * However, if a scope is active, then the APM agent can capture additional information, so this method
      * exists to make it possible to use scopes in the few situation where it makes sense.
      *
-     * @param spanId the ID of a currently-open span for which to open a scope.
+     * @param traceable provides the ID of a currently-open span for which to open a scope.
      * @return a method to close the scope when you are finished with it.
      */
     @Override
-    public Releasable withScope(SpanId spanId) {
-        final Context context = spans.get(spanId);
+    public Releasable withScope(Traceable traceable) {
+        final Context context = spans.get(traceable.getSpanId());
         if (context != null) {
             var scope = AccessController.doPrivileged((PrivilegedAction<Scope>) context::makeCurrent);
             return scope::close;
@@ -337,50 +338,50 @@ private void setSpanAttributes(ThreadContext threadContext, @Nullable Map<String
     }
 
     @Override
-    public void addError(SpanId spanId, Throwable throwable) {
-        final var span = Span.fromContextOrNull(spans.get(spanId));
+    public void addError(Traceable traceable, Throwable throwable) {
+        final var span = Span.fromContextOrNull(spans.get(traceable.getSpanId()));
         if (span != null) {
             span.recordException(throwable);
         }
     }
 
     @Override
-    public void setAttribute(SpanId spanId, String key, boolean value) {
-        final var span = Span.fromContextOrNull(spans.get(spanId));
+    public void setAttribute(Traceable traceable, String key, boolean value) {
+        final var span = Span.fromContextOrNull(spans.get(traceable.getSpanId()));
         if (span != null) {
             span.setAttribute(key, value);
         }
     }
 
     @Override
-    public void setAttribute(SpanId spanId, String key, double value) {
-        final var span = Span.fromContextOrNull(spans.get(spanId));
+    public void setAttribute(Traceable traceable, String key, double value) {
+        final var span = Span.fromContextOrNull(spans.get(traceable.getSpanId()));
         if (span != null) {
             span.setAttribute(key, value);
         }
     }
 
     @Override
-    public void setAttribute(SpanId spanId, String key, long value) {
-        final var span = Span.fromContextOrNull(spans.get(spanId));
+    public void setAttribute(Traceable traceable, String key, long value) {
+        final var span = Span.fromContextOrNull(spans.get(traceable.getSpanId()));
         if (span != null) {
             span.setAttribute(key, value);
         }
     }
 
     @Override
-    public void setAttribute(SpanId spanId, String key, String value) {
-        final var span = Span.fromContextOrNull(spans.get(spanId));
+    public void setAttribute(Traceable traceable, String key, String value) {
+        final var span = Span.fromContextOrNull(spans.get(traceable.getSpanId()));
         if (span != null) {
             span.setAttribute(key, value);
         }
     }
 
     @Override
-    public void stopTrace(SpanId spanId) {
-        final var span = Span.fromContextOrNull(spans.remove(spanId));
+    public void stopTrace(Traceable traceable) {
+        final var span = Span.fromContextOrNull(spans.remove(traceable.getSpanId()));
         if (span != null) {
-            logger.trace("Finishing trace [{}]", spanId);
+            logger.trace("Finishing trace [{}]", traceable);
             AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
                 span.end();
                 return null;
@@ -400,8 +401,8 @@ public void stopTrace() {
     }
 
     @Override
-    public void addEvent(SpanId spanId, String eventName) {
-        final var span = Span.fromContextOrNull(spans.get(spanId));
+    public void addEvent(Traceable traceable, String eventName) {
+        final var span = Span.fromContextOrNull(spans.get(traceable.getSpanId()));
         if (span != null) {
             span.addEvent(eventName);
         }
@@ -425,7 +426,7 @@ private static boolean isSupportedContextKey(String key) {
     }
 
     // VisibleForTesting
-    Map<SpanId, Context> getSpans() {
+    Map<String, Context> getSpans() {
         return spans;
     }