Merge branch 'main' into makeContentOptionalUnifiedInferenceApi

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -24,12 +24,17 @@ allprojects {
   }
 }
 
+interface Injected {
+  @Inject FileSystemOperations getFs()
+}
+
 // Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE
 if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin)
 
   def elasticsearchProject = locateElasticsearchWorkspace(gradle)
 
+  def rootFolder = project.rootDir
   tasks.register('configureIdeCheckstyle') {
     group = 'ide'
     description = 'Generated a suitable checkstyle config for IDEs'
@@ -39,10 +44,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     String checkstyleConfig = "${resources}/checkstyle.xml"
     String checkstyleSuppressions = "${resources}/checkstyle_suppressions.xml"
     String checkstyleIdeFragment = "${resources}/checkstyle_ide_fragment.xml"
-    String checkstyleIdeConfig = "${rootDir}/checkstyle_ide.xml"
+    String checkstyleIdeConfig = "${rootFolder}/checkstyle_ide.xml"
 
     String checkstylePluginConfigTemplate = "${resources}/checkstyle-idea.xml"
-    String checkstylePluginConfig = "${rootDir}/.idea/checkstyle-idea.xml"
+    String checkstylePluginConfig = "${rootFolder}/.idea/checkstyle-idea.xml"
 
     inputs.files(
       file(checkstyleConfig),
@@ -53,31 +58,33 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       file(checkstyleIdeConfig),
       file(checkstylePluginConfig)
     )
+    def injected = project.objects.newInstance(Injected)
 
+    def projectFolder = project.layout.projectDirectory.asFile
     doLast {
       // Configure the IntelliJ Checkstyle plugin by copying a standard file. We don't simply commit
       // the result to version control, because the plugin has a habit of modifying the file and
       // replacing the `$PROJECT_DIR$` placeholders, which developers must then revert.
-      project.copy {
+      injected.fs.copy {
         from(checkstylePluginConfigTemplate)
-        into("${rootDir}/.idea")
+        into("${rootFolder}/.idea")
         expand(jarLocation: buildConventionsJar, configLocation: checkstyleIdeConfig)
       }
 
       // Create an IDE-specific checkstyle config by first copying the standard config
       Files.copy(
-        Paths.get(file(checkstyleConfig).getPath()),
-        Paths.get(file(checkstyleIdeConfig).getPath()),
+        Paths.get(new File(checkstyleConfig).getPath()),
+        Paths.get(new File(checkstyleIdeConfig).getPath()),
         StandardCopyOption.REPLACE_EXISTING
       )
 
       // There are some rules that we only want to enable in an IDE. These
       // are extracted to a separate file, and merged into the IDE-specific
       // Checkstyle config.
-      Node xmlFragment = parseXml(checkstyleIdeFragment)
+      Node xmlFragment = IdeaXmlUtil.parseXml(checkstyleIdeFragment)
 
       // Edit the copy so that IntelliJ can copy with it
-      modifyXml(checkstyleIdeConfig, { xml ->
+      IdeaXmlUtil.modifyXml(checkstyleIdeConfig, { xml ->
         // Add all the nodes from the fragment file
         Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' }
         xmlFragment.module.each { treeWalker.append(it) }
@@ -103,7 +110,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Configures the appropriate JVM for Gradle'
 
     doLast {
-      modifyXml('.idea/gradle.xml') { xml ->
+      IdeaXmlUtil.modifyXml('.idea/gradle.xml') { xml ->
         def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings
         // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle
         gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) }
@@ -127,7 +134,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enable per-module *.iml files'
 
     doLast {
-      modifyXml('.idea/misc.xml') {xml ->
+      IdeaXmlUtil.modifyXml('.idea/misc.xml') {xml ->
         def externalStorageConfig = xml.component.find { it.'@name' == 'ExternalStorageConfigurationManager' }
         if (externalStorageConfig) {
           xml.remove(externalStorageConfig)
@@ -142,13 +149,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
 
-    ext {
-      enablePreview = { moduleFile, languageLevel ->
-        modifyXml(moduleFile) { xml ->
+//    ext {
+      def enablePreview = { moduleFile, languageLevel ->
+        IdeaXmlUtil.modifyXml(moduleFile) { xml ->
           xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
         }
       }
-    }
+//    }
 
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
@@ -278,33 +285,37 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
  * @param preface optional front matter to add after the XML declaration
  * but before the XML document, e.g. a doctype or comment
  */
-void modifyXml(Object path, Action<? super Node> action, String preface = null) {
-  if (project.file(path).exists()) {
-    Node xml = parseXml(path)
-    action.execute(xml)
-
-    File xmlFile = project.file(path)
-    xmlFile.withPrintWriter { writer ->
-      def printer = new XmlNodePrinter(writer)
-      printer.namespaceAware = true
-      printer.preserveWhitespace = true
-      writer.write("<?xml version=\"1.0\"?>\n")
-
-      if (preface != null) {
-        writer.write(preface)
+
+class IdeaXmlUtil {
+  static Node parseXml(Object xmlPath) {
+    File xmlFile = new File(xmlPath)
+    XmlParser xmlParser = new XmlParser(false, true, true)
+    xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
+    Node xml = xmlParser.parse(xmlFile)
+    return xml
+  }
+
+  static void modifyXml(Object xmlPath, Action<? super Node> action, String preface = null) {
+    File xmlFile = new File(xmlPath)
+    if (xmlFile.exists()) {
+      Node xml = parseXml(xmlPath)
+      action.execute(xml)
+
+      xmlFile.withPrintWriter { writer ->
+        def printer = new XmlNodePrinter(writer)
+        printer.namespaceAware = true
+        printer.preserveWhitespace = true
+        writer.write("<?xml version=\"1.0\"?>\n")
+
+        if (preface != null) {
+          writer.write(preface)
+        }
+        printer.print(xml)
       }
-      printer.print(xml)
     }
   }
 }
 
-Node parseXml(Object path) {
-  File xmlFile = project.file(path)
-  XmlParser xmlParser = new XmlParser(false, true, true)
-  xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
-  Node xml = xmlParser.parse(xmlFile)
-  return xml
-}
 
 Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
   if (gradle.parent == null) {

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -1,21 +1,17 @@
 ---
 apiVersion: v1
-
 # The repository name in registry1, excluding /ironbank/
 name: "elastic/elasticsearch/elasticsearch"
-
 # List of tags to push for the repository in registry1
 # The most specific version should be the first tag and will be shown
 # on ironbank.dsop.io
 tags:
 - "${version}"
 - "latest"
-
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.4"
-
+  BASE_TAG: "9.5"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"
@@ -34,7 +30,6 @@ labels:
   mil.dso.ironbank.image.type: "commercial"
   # Product the image belongs to for grouping multiple images
   mil.dso.ironbank.product.name: "elasticsearch"
-
 # List of resources to make available to the offline build context
 resources:
   - filename: "elasticsearch-${version}-linux-x86_64.tar.gz"
@@ -47,7 +42,6 @@ resources:
     validation:
       type: "sha256"
       value: "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"
-
 # List of project maintainers
 maintainers:
   - name: "Mark Vieira"

docs/changelog/118016.yaml

@@ -0,0 +1,6 @@
+pr: 118016
+summary: Propagate status codes from shard failures appropriately
+area: Search
+type: enhancement
+issues:
+ - 118482

docs/changelog/118802.yaml

@@ -0,0 +1,5 @@
+pr: 118802
+summary: ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape
+area: "ES|QL"
+type: enhancement
+issues: []

docs/changelog/118870.yaml

@@ -0,0 +1,6 @@
+pr: 118870
+summary: Rewrite TO_UPPER/TO_LOWER comparisons
+area: ES|QL
+type: enhancement
+issues:
+ - 118304

docs/changelog/118999.yaml

@@ -0,0 +1,6 @@
+pr: 118999
+summary: Fix loss of context in the inference API for streaming APIs
+area: Machine Learning
+type: bug
+issues:
+ - 119000

docs/changelog/119131.yaml

@@ -0,0 +1,5 @@
+pr: 119131
+summary: Expose BwC enrich cache setting in plugin
+area: Ingest Node
+type: bug
+issues: []

docs/reference/indices/shard-stores.asciidoc

@@ -172,9 +172,8 @@ The API returns the following response:
                             "attributes": {},
                             "roles": [...],
                             "version": "8.10.0",
-                            "min_index_version": 8000099,
-                            "min_read_only_index_version": 7000099,
-                            "max_index_version": 9004000
+                            "min_index_version": 7000099,
+                            "max_index_version": 8100099
                         },
                         "allocation_id": "2iNySv_OQVePRX-yaRH_lQ", <4>
                         "allocation" : "primary|replica|unused" <5>
@@ -194,7 +193,6 @@ The API returns the following response:
 // TESTRESPONSE[s/"roles": \[[^]]*\]/"roles": $body.$_path/]
 // TESTRESPONSE[s/"8.10.0"/\$node_version/]
 // TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/]
-// TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/]
 // TESTRESPONSE[s/"max_index_version": 8100099/"max_index_version": $body.$_path/]
 
 

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java

@@ -55,7 +55,7 @@ static CheckAction forPlugins(Runnable action) {
         entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
         entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
         entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)),
-        // entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
+        entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
         entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline))
     );
 
@@ -78,7 +78,11 @@ private static void createClassLoader() {
     }
 
     private static void processBuilder_start() {
-        // TODO: processBuilder().start();
+        try {
+            new ProcessBuilder("").start();
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
     }
 
     private static void processBuilder_startPipeline() {

modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java

@@ -99,17 +99,12 @@ public void setUp() throws Exception {
                 .setSettings(
                     settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.NEW_INDEXVERSION_FORMAT).build()
                 )
-                .setMapping(mapping)
-                .get()
-        );
-
-        assertAcked(
+                .setMapping(mapping),
             indicesAdmin().prepareCreate(afterIndex)
                 .setSettings(
                     settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.TIME_SERIES_ID_HASHING).build()
                 )
                 .setMapping(mapping)
-                .get()
         );
 
         final TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset();

modules/analysis-common/build.gradle

@@ -37,3 +37,16 @@ artifacts {
 tasks.named("yamlRestCompatTestTransform").configure { task ->
   task.replaceValueInMatch("tokens.0.token", "absenț", "romanian")
 }
+
+tasks.named("yamlRestTest").configure {
+  if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 24 ||
+    "-Des.entitlements.enabled=true".equals(System.getProperty("tests.jvm.argline"))) {
+    systemProperty 'tests.rest.blacklist',
+      [
+        // AWAITSFIX: this test relies on security manager, which doesn't exist in JDK 24.
+        // and entitlements don't yet replace the functionality.
+        // see https://github.com/elastic/elasticsearch/issues/119130
+        'analysis-common/40_token_filters/stemmer_override file access',
+      ].join(',')
+  }
+}

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java

@@ -89,8 +89,6 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final String language;
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(StemmerTokenFilterFactory.class);
-
     StemmerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
         super(name);
         this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter")));
@@ -192,7 +190,7 @@ public boolean incrementToken() {
             } else if ("german".equalsIgnoreCase(language)) {
                 return new SnowballFilter(tokenStream, new GermanStemmer());
             } else if ("german2".equalsIgnoreCase(language)) {
-                DEPRECATION_LOGGER.critical(
+                deprecationLogger.critical(
                     DeprecationCategory.ANALYSIS,
                     "german2_stemmer_deprecation",
                     "The 'german2' stemmer has been deprecated and folded into the 'german' Stemmer. "

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java

@@ -13,7 +13,6 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.synonym.SynonymFilter;
 import org.apache.lucene.analysis.synonym.SynonymMap;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService.IndexCreationContext;
@@ -130,8 +129,6 @@ public static SynonymsSource fromSettings(Settings settings) {
         }
     }
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(SynonymTokenFilterFactory.class);
-
     private final String format;
     private final boolean expand;
     private final boolean lenient;

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/20_unsupported_apis.yml

@@ -45,8 +45,6 @@
   - do:
       indices.close:
         index: logs-*
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
   - length: { indices: 0 }
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/40_supported_apis.yml

@@ -230,8 +230,6 @@ teardown:
   - do:
       indices.close:
         index: ".ds-simple-data-stream1-*000001,.ds-simple-data-stream1-*000002"
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
   - is_true: acknowledged
 
   - do:

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml

@@ -54,8 +54,6 @@ setup:
   - do:
       indices.close:
         index: test_index2
-      allowed_warnings:
-        - "the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour"
 
   - do:
       indices.create: