Merge branch '8.x' into backport/8.x/pr-117277

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.26", "8.16.1", "8.17.0", "8.18.0"]
+            BWC_VERSION: ["7.17.26", "8.16.2", "8.17.0", "8.18.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/periodic-packaging.yml

@@ -9,6 +9,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -575,8 +576,8 @@ steps:
         env:
           BWC_VERSION: 8.15.4
 
-      - label: "{{matrix.image}} / 8.16.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.1
+      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -589,7 +590,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.1
+          BWC_VERSION: 8.16.2
 
       - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0

.buildkite/pipelines/periodic-platform-support.yml

@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/periodic.yml

@@ -648,8 +648,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.1#bwcTest
+      - label: 8.16.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -658,7 +658,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.1
+          BWC_VERSION: 8.16.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.26", "8.16.1", "8.17.0", "8.18.0"]
+            BWC_VERSION: ["7.17.26", "8.16.2", "8.17.0", "8.18.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -819,7 +819,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.26", "8.16.1", "8.17.0", "8.18.0"]
+            BWC_VERSION: ["7.17.26", "8.16.2", "8.17.0", "8.18.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -11,6 +11,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -38,6 +39,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -65,6 +67,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/scripts/dra-workflow.sh

@@ -6,7 +6,7 @@ WORKFLOW="${DRA_WORKFLOW:-snapshot}"
 BRANCH="${BUILDKITE_BRANCH:-}"
 
 # Don't publish main branch to staging
-if [[ "$BRANCH" == "main" && "$WORKFLOW" == "staging" ]]; then
+if [[ ("$BRANCH" == "main" || "$BRANCH" == *.x) && "$WORKFLOW" == "staging" ]]; then
   exit 0
 fi
 

.ci/bwcVersions

@@ -33,6 +33,6 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.4"
-  - "8.16.1"
+  - "8.16.2"
   - "8.17.0"
   - "8.18.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.26"
-  - "8.16.1"
+  - "8.16.2"
   - "8.17.0"
   - "8.18.0"

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -24,12 +24,12 @@ if (BuildParams.inFipsJvm) {
     File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename)
     File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy')
     File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks')
-    def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.4')
-    def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.17')
+    def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.5')
+    def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19')
     def manualDebug = false; //change this to manually debug bouncy castle in an IDE
     if(manualDebug) {
-      bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.4')
-      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.17'){
+      bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.5')
+      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19'){
         exclude group: 'org.bouncycastle', module: 'bc-fips'  // to avoid jar hell
       }
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -20,9 +20,12 @@
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.api.tasks.javadoc.Javadoc;
 import org.gradle.api.tasks.testing.Test;
+import org.gradle.external.javadoc.CoreJavadocOptions;
 import org.gradle.jvm.tasks.Jar;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -79,6 +82,7 @@ public void apply(Project project) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
                 SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
                 configureSourceSetInJar(project, mainSourceSet, javaVersion);
+                addJar(project, mainSourceSet, javaVersion);
                 mainSourceSets.add(mainSourceSetName);
                 testSourceSets.add(mainSourceSetName);
 
@@ -142,6 +146,29 @@ private SourceSet addSourceSet(
         return sourceSet;
     }
 
+    private void addJar(Project project, SourceSet sourceSet, int javaVersion) {
+        project.getConfigurations().register("java" + javaVersion);
+        TaskProvider<Jar> jarTask = project.getTasks().register("java" + javaVersion + "Jar", Jar.class, task -> {
+            task.from(sourceSet.getOutput());
+        });
+        project.getArtifacts().add("java" + javaVersion, jarTask);
+    }
+
+    private void configurePreviewFeatures(Project project, SourceSet sourceSet, int javaVersion) {
+        project.getTasks().withType(JavaCompile.class).named(sourceSet.getCompileJavaTaskName()).configure(compileTask -> {
+            CompileOptions compileOptions = compileTask.getOptions();
+            compileOptions.getCompilerArgs().add("--enable-preview");
+            compileOptions.getCompilerArgs().add("-Xlint:-preview");
+
+            compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); });
+        });
+        project.getTasks().withType(Javadoc.class).named(name -> name.equals(sourceSet.getJavadocTaskName())).configureEach(javadocTask -> {
+            CoreJavadocOptions options = (CoreJavadocOptions) javadocTask.getOptions();
+            options.addBooleanOption("-enable-preview", true);
+            options.addStringOption("-release", String.valueOf(javaVersion));
+        });
+    }
+
     private void configureSourceSetInJar(Project project, SourceSet sourceSet, int javaVersion) {
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
         jarTask.configure(task -> task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput())));

build-tools-internal/src/main/resources/fips_java.policy

@@ -5,6 +5,7 @@ grant {
      permission java.security.SecurityPermission "getProperty.jdk.tls.disabledAlgorithms";
      permission java.security.SecurityPermission "getProperty.jdk.certpath.disabledAlgorithms";
      permission java.security.SecurityPermission "getProperty.jdk.tls.server.defaultDHEParameters";
+     permission java.security.SecurityPermission "getProperty.org.bouncycastle.ec.max_f2m_field_size";
      permission java.lang.RuntimePermission "getProtectionDomain";
      permission java.util.PropertyPermission "java.runtime.name", "read";
      permission org.bouncycastle.crypto.CryptoServicesPermission "tlsAlgorithmsEnabled";
@@ -20,6 +21,6 @@ grant {
 };
 
 // rely on the caller's socket permissions, the JSSE TLS implementation here is always allowed to connect
-grant codeBase "file:${jdk.module.path}/bctls-fips-1.0.17.jar" {
+grant codeBase "file:${jdk.module.path}/bctls-fips-1.0.19.jar" {
   permission java.net.SocketPermission "*", "connect";
 };

distribution/packages/build.gradle

@@ -335,7 +335,6 @@ Closure commonDebConfig(String architecture) {
 
     // versions found on oldest supported distro, centos-6
     requires('bash', '4.1', GREATER | EQUAL)
-    requires('lsb-base', '4', GREATER | EQUAL)
     requires 'libc6'
     requires 'adduser'
 

distribution/packages/src/deb/lintian/elasticsearch

@@ -5,8 +5,6 @@
 changelog-file-missing-in-native-package
 
 # we intentionally copy our copyright file for all deb packages
-copyright-file-contains-full-apache-2-license
-copyright-not-using-common-license-for-apache2
 copyright-without-copyright-notice
 
 # we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
@@ -16,37 +14,23 @@ arch-dependent-file-in-usr-share
 missing-dep-on-jarwrapper
 
 # we prefer to not make our config and log files world readable
-non-standard-file-perm etc/default/elasticsearch 0660 != 0644
-non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
-non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
-non-standard-file-perm etc/elasticsearch/*
-non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
-non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
-
-# this lintian tag is simply wrong; contrary to the explanation, Debian systemd
-# does actually look at /usr/lib/systemd/system
-systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service
+non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch]
+non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/]
+non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/]
+non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*]
+non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/]
+non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/]
 
 # the package scripts handle systemd directly and don't need to use deb helpers
 maintainer-script-calls-systemctl
 
 # bundled JDK
 embedded-library
-unstripped-binary-or-object usr/share/elasticsearch/jdk/*
-extra-license-file usr/share/elasticsearch/jdk/legal/*
-hardening-no-pie usr/share/elasticsearch/jdk/bin/*
-hardening-no-pie usr/share/elasticsearch/jdk/lib/*
+unstripped-binary-or-object [usr/share/elasticsearch/jdk/*]
 
 # the system java version that lintian assumes is far behind what elasticsearch uses
 unknown-java-class-version
 
-# elastic licensed modules contain elastic license
-extra-license-file usr/share/elasticsearch/modules/*
-
-# This dependency appears to have a packaging flaw, and includes a
-# generated source file alongside the compiled version
-jar-contains-source usr/share/elasticsearch/modules/repository-gcs/api-common*.jar *
-
 # There's no `License` field in Debian control files, but earlier versions
 # of `lintian` were more permissive. Override this warning so that we can
 # run `lintian` on different releases of Debian. The format of this override
@@ -58,8 +42,27 @@ unknown-field License
 # indirectly to libc via libdl. This might not be best practice but we
 # don't build them ourselves and the license precludes us modifying them
 # to fix this.
-library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so
+library-not-linked-against-libc [usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*]
+
+
+# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11)
+# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with
+# duplication and ignoring some issues in the test code.
+
+
+# we prefer to not make our config and log files world readable
+non-standard-file-perm etc/default/elasticsearch 0660 != 0644
+non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
+non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
+non-standard-file-perm etc/elasticsearch/*
+non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
+non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
 
-# shared-lib-without-dependency-information (now shared-library-lacks-prerequisites) is falsely reported for libvec.so
-# which has no dependencies (not even libc) besides the symbols in the base executable.
-shared-lib-without-dependency-information usr/share/elasticsearch/lib/platform/linux-x64/libvec.so
+# bundled JDK
+unstripped-binary-or-object usr/share/elasticsearch/jdk/*
+
+# Intel MKL libraries are not linked directly to libc. They are linked
+# indirectly to libc via libdl. This might not be best practice but we
+# don't build them ourselves and the license precludes us modifying them
+# to fix this.
+library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*

distribution/tools/plugin-cli/build.gradle

@@ -29,7 +29,7 @@ dependencies {
   implementation 'org.ow2.asm:asm-tree:9.7'
 
   api "org.bouncycastle:bcpg-fips:1.0.7.1"
-  api "org.bouncycastle:bc-fips:1.0.2.4"
+  api "org.bouncycastle:bc-fips:1.0.2.5"
   testImplementation project(":test:framework")
   testImplementation "com.google.jimfs:jimfs:${versions.jimfs}"
   testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}"

docs/changelog/112989.yaml

@@ -0,0 +1,5 @@
+pr: 112989
+summary: Upgrade Bouncy Castle FIPS dependencies
+area: Security
+type: upgrade
+issues: []

docs/changelog/115616.yaml

@@ -0,0 +1,6 @@
+pr: 115616
+summary: Fix double lookup failure on ESQL
+area: ES|QL
+type: bug
+issues:
+  - 111398

docs/changelog/115836.yaml

@@ -0,0 +1,5 @@
+pr: 115836
+summary: Catch and handle disconnect exceptions in search
+area: Search
+type: bug
+issues: []

docs/changelog/116765.yaml

@@ -0,0 +1,5 @@
+pr: 116765
+summary: Metrics for incremental bulk splits
+area: Distributed
+type: enhancement
+issues: []

docs/changelog/116809.yaml

@@ -0,0 +1,5 @@
+pr: 116809
+summary: "Distinguish `LicensedFeature` by family field"
+area: License
+type: bug
+issues: []

docs/changelog/117243.yaml

@@ -0,0 +1,5 @@
+pr: 117243
+summary: Bump major version for feature migration system indices
+area: Infra/Core
+type: upgrade
+issues: []

docs/changelog/117287.yaml

@@ -0,0 +1,5 @@
+pr: 117287
+summary: Fixing bug setting index when parsing Google Vertex AI results
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/117294.yaml

@@ -0,0 +1,5 @@
+pr: 117294
+summary: Always Emit Inference ID in Semantic Text Mapping
+area: Mapping
+type: bug
+issues: []

docs/changelog/117316.yaml

@@ -0,0 +1,5 @@
+pr: 117316
+summary: Fix validation of SORT by aggregate functions
+area: ES|QL
+type: bug
+issues: []

docs/reference/esql/functions/aggregation-functions.asciidoc

@@ -20,7 +20,7 @@ The <<esql-stats-by>> command supports these aggregate functions:
 * <<esql-sum>>
 * <<esql-top>>
 * <<esql-values>>
-* experimental:[] <<esql-weighted_avg>>
+* <<esql-weighted_avg>>
 // end::agg_list[]
 
 include::layout/avg.asciidoc[]